Let's Write a Qi Compiler!

.github/workflows/benchmarks.yml

@@ -25,7 +25,7 @@ jobs:
         run: make install-sdk
       - name: Run benchmark
         shell: 'bash --noprofile --norc -eo pipefail {0}'
-        run: make report-benchmarks | tee benchmarks.txt
+        run: make performance-report | tee benchmarks.txt
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: true
       matrix:
         racket-variant: ['BC', 'CS']
-        racket-version: ['8.3', 'stable']
+        racket-version: ['8.5', 'stable']
         experimental: [false]
         include:
           - racket-version: 'current'

Makefile

@@ -7,12 +7,15 @@ DEPS-FLAGS=--check-pkg-deps --unused-pkg-deps
 
 help:
 	@echo "install - install package along with dependencies"
+	@echo "install-sdk - install the SDK which includes developer tools"
 	@echo "remove - remove package"
+	@echo "remove-sdk - remove SDK; this will not remove SDK dependencies"
 	@echo "build - Compile libraries"
 	@echo "build-docs - Build docs"
 	@echo "build-standalone-docs - Build self-contained docs that could be hosted somewhere"
 	@echo "build-all - Compile libraries, build docs, and check dependencies"
 	@echo "clean - remove all build artifacts"
+	@echo "clean-sdk - remove all build artifacts in SDK paths"
 	@echo "check-deps - check dependencies"
 	@echo "test - run tests"
 	@echo "test-with-errortrace - run tests with error tracing"
@@ -27,6 +30,8 @@ help:
 	@echo "  definitions"
 	@echo "  macro"
 	@echo "  util"
+	@echo "  expander"
+	@echo "  compiler"
 	@echo "  probe"
 	@echo "    Note: As probe is not in qi-lib, it isn't part of"
 	@echo "    the tests run in the 'test' target."
@@ -37,9 +42,14 @@ help:
 	@echo "docs - view docs in a browser"
 	@echo "profile - Run comprehensive performance benchmarks"
 	@echo "profile-competitive - Run competitive benchmarks"
-	@echo "profile-forms - Run benchmarks for individual Qi forms"
+	@echo "profile-local - Run benchmarks for individual Qi forms"
+	@echo "profile-nonlocal - Run nonlocal benchmarks exercising many components at once"
 	@echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)"
-	@echo "report-benchmarks - Run benchmarks for Qi forms and produce results for use in CI"
+	@echo "performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression"
+	@echo "  For use in regression: make performance-report > /path/to/before.json"
+	@echo "performance-regression-report - Run benchmarks for Qi forms against a reference report."
+	@echo "  make performance-regression-report REF=/path/to/before.json"
+
 
 # Primarily for use by CI.
 # Installs dependencies as well as linking this as a package.
@@ -82,36 +92,47 @@ build-standalone-docs:
 clean:
 	raco setup --fast-clean --pkgs $(PACKAGE-NAME)-{lib,test,doc,probe}
 
+clean-sdk:
+	raco setup --fast-clean --pkgs $(PACKAGE-NAME)-sdk
+
 # Primarily for use by CI, after make install -- since that already
 # does the equivalent of make setup, this tries to do as little as
 # possible except checking deps.
 check-deps:
 	raco setup --no-docs $(DEPS-FLAGS) $(PACKAGE-NAME)
 
 # Suitable for both day-to-day dev and CI
+# Note: we don't test qi-doc since there aren't any tests there atm
+# and it also seems to make things extremely slow to include it.
 test:
-	raco test -exp $(PACKAGE-NAME)-{lib,test,doc,probe}
+	raco test -exp $(PACKAGE-NAME)-{lib,test,probe}
 
 test-flow:
-	racket $(PACKAGE-NAME)-test/tests/flow.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/flow.rkt
 
 test-on:
-	racket $(PACKAGE-NAME)-test/tests/on.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/on.rkt
 
 test-threading:
-	racket $(PACKAGE-NAME)-test/tests/threading.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/threading.rkt
 
 test-switch:
-	racket $(PACKAGE-NAME)-test/tests/switch.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/switch.rkt
 
 test-definitions:
-	racket $(PACKAGE-NAME)-test/tests/definitions.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/definitions.rkt
 
 test-macro:
-	racket $(PACKAGE-NAME)-test/tests/macro.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/macro.rkt
 
 test-util:
-	racket $(PACKAGE-NAME)-test/tests/util.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/util.rkt
+
+test-expander:
+	racket -y $(PACKAGE-NAME)-test/tests/expander.rkt
+
+test-compiler:
+	racket -y $(PACKAGE-NAME)-test/tests/compiler.rkt
 
 test-probe:
 	raco test -exp $(PACKAGE-NAME)-probe
@@ -159,20 +180,27 @@ cover: coverage-check coverage-report
 cover-coveralls:
 	raco cover -b -f coveralls -p $(PACKAGE-NAME)-{lib,test}
 
-profile-forms:
-	echo "Profiling forms..."
-	racket $(PACKAGE-NAME)-sdk/profile/forms.rkt
+profile-local:
+	racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt
+
+profile-loading:
+	racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt
 
 profile-selected-forms:
-	@echo "Use 'racket profile/forms.rkt' directly, with -f form-name for each form."
+	@echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -s form-name for each form."
 
 profile-competitive:
-	echo "Running competitive benchmarks..."
-	racket $(PACKAGE-NAME)-sdk/profile/competitive.rkt
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-competitive.rkt
+
+profile-nonlocal:
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-intrinsic.rkt -l qi
+
+profile: profile-local profile-nonlocal profile-loading
 
-profile: profile-competitive profile-forms
+performance-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -f json
 
-report-benchmarks:
-	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt
+performance-regression-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF)
 
-.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile report-benchmarks
+.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-expander test-compiler test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-local profile-loading profile-selected-forms profile-competitive profile-nonlocal profile performance-report performance-regression-report

qi-doc/scribblings/eval.rkt

@@ -0,0 +1,30 @@
+#lang racket/base
+
+(provide make-eval-for-docs)
+
+(require racket/sandbox)
+
+(define (make-eval-for-docs . exprs)
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (apply make-evaluator
+              'racket/base
+              '(require qi
+                        qi/probe
+                        (only-in racket/list range first rest)
+                        racket/format
+                        racket/string
+                        (only-in racket/function curry)
+                        (for-syntax syntax/parse
+                                    racket/base))
+              '(define (sqr x)
+                 (* x x))
+              '(define ->string number->string)
+              exprs)))))

qi-doc/scribblings/field-guide.scrbl

@@ -2,24 +2,12 @@
 @require[scribble/manual
          scribble-abbrevs/manual
          scribble/example
-         racket/sandbox
+		 "eval.rkt"
          @for-label[qi
                     qi/probe
                     racket]]
 
-@(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              qi/probe
-                              (only-in racket/list range)
-                              racket/string
-                              (for-syntax syntax/parse
-                                          racket/base))
-                    '(define (sqr x)
-                       (* x x)))))
+@(define eval-for-docs (make-eval-for-docs))
 
 @title{Field Guide}
 
@@ -41,6 +29,10 @@ Decompose your @tech{flow} into its smallest components, and name each so that t
 
 A journeyman of one's craft -- a woodworker, electrician, or a plumber, say -- always goes to work with a trusty toolbox that contains the tools of the trade, some perhaps even of their own design. An electrician, for instance, may have a voltage tester, a multimeter, and a continuity tester in her toolbox. Although these are "debugging" tools, they aren't just for identifying bugs -- by providing rapid feedback, they enable her to explore and find creative solutions quickly and reliably. It's the same with Qi. Learn to use the @seclink["Debugging"]{debugging tools}, and use them often.
 
+@subsection{Be Intentional About Effects}
+
+Qi encourages a style that avoids "accidental" effects. A flow should either be pure (that is, it should be free of "side effects" such as printing to the screen or writing to a file), or its entire purpose should be to fulfill a side effect. It is considered inadvisable to have a function with sane inputs and outputs (resembling a pure function) that also performs a side effect. It would be better to decouple the effect from the rest of your function (@seclink["Use_Small_Building_Blocks"]{splitting it into smaller functions}, as necessary) and perform the effect explicitly via the @racket[effect] form, or otherwise escape from Qi using something like @racket[esc] (note that @seclink["Identifiers"]{function identifiers} used in a flow context are implicitly @racket[esc]aped) in order to perform the effect. This will ensure that there are no surprises with regard to @seclink["Order_of_Effects"]{order of effects}.
+
 @section{Debugging}
 
 There are three prominent debugging strategies which may be used independently or in tandem -- @seclink["Using_Side_Effects"]{side effects}, @seclink["Using_a_Probe"]{probing}, and @seclink["Using_Fixtures"]{fixtures}.
@@ -208,6 +200,17 @@ Methodical use of @racket[gen] together with the @seclink["Using_a_Probe"]{probe
 
 @bold{Common example}: Syntax patterns are defined in the @seclink["stxparse" #:doc '(lib "syntax/scribblings/syntax.scrbl")]{syntax/parse} library. If you are using them in Qi macros, you will need to @racket[(require syntax/parse)] at the appropriate phase level.
 
+@subsubsection{Identifier's Binding is Ambiguous}
+
+@codeblock{
+; count: identifier's binding is ambiguous
+;   in: count
+}
+
+@bold{Meaning}: The @tech/guide{expander} attempted to resolve a @tech/reference{reference} and found more than one possible @tech/reference{binding}.
+
+@bold{Common example}: Having a Racket function in scope that has the same name as a Qi form, and attempting to use this @seclink["Identifiers"]{unqualified identifier} as a flow. To avoid the issue, rename the Racket function to something else, or use an explicit @racket[esc] to indicate the Racket binding.
+
 @subsubsection{Not Defined as Syntax Class}
 
 @codeblock{
@@ -322,7 +325,7 @@ But in an idle moment, this clever shortcut may tempt you:
     (~> (3) ((get-f 1)))
   ]
 
-That is, since Qi typically interprets parenthesized expressions as @seclink["Templates_and_Partial_Application"]{partial application templates}, you might expect that this would pass the value @racket[3] to the function resulting from @racket[(get-f 1)]. In fact, that isn't what happens, and an error is raised instead. As there is only one datum within the outer pair of parentheses in @racket[((get-f 1))], the usual interpretation as partial application would not be useful, and could even lead to unexpected behavior (at least, with the current implementation that uses Racket's @racket[curry]). So instead, Qi attempts to interpret the expression as written, that is, as if it were wrapped in @racket[esc]. As a result, it attempts to evaluate @racket[((get-f 1))] and expects to receive a value that can be used as a @tech{flow} here. If, as in the above expression, the function resulting from @racket[(get-f 1)] expects a single argument, this is now an error as it is being invoked with none.
+That is, since Qi typically interprets parenthesized expressions as @seclink["Templates_and_Partial_Application"]{partial application templates}, you might expect that this would pass the value @racket[3] to the function resulting from @racket[(get-f 1)]. In fact, that isn't what happens, and an error is raised instead. As there is only one datum within the outer pair of parentheses in @racket[((get-f 1))], the usual interpretation as partial application would not typically be useful, so Qi opts to treat it as invalid syntax.
 
 One way to dodge this is by using an explicit template:
 
@@ -332,14 +335,6 @@ One way to dodge this is by using an explicit template:
 
 This works in most cases, but it has different semantics than the version using @racket[esc], as that version evaluates the escaped expression first to yield the @tech{flow} that will be applied to inputs, while this one only evaluates the (up to that point, incomplete) expression when it is actually invoked with arguments. In the most common cases there will be no difference to the result, but if the flow is invoked multiple times (for instance, if it were first defined as @racket[(define-flow my-flow (☯ ((get-f 1) _)))]), then the expression too would be evaluated multiple times, producing different functions each time. This may be computationally more expensive than using @racket[esc], and also, if either @racket[get-f] or the function it produces is stateful in any way (for instance, if it is a @hyperlink["https://www.gnu.org/software/guile/manual/html_node/Closure.html"]{closure} or if there is any randomness involved), then this version would also produce different results than the @racket[esc] version.
 
-Another way to do it is to simply promote the expression out of the nest:
-
-@racketblock[
-    (~> (3) (get-f 1))
-  ]
-
-Now, you might, once again, expect this to be treated as a partial application template, so that this would be equivalent to @racket[(get-f 3 1)] and would raise an error. But in fact, since the expression @racket[(get-f 1)] happens to be fully qualified with all the arguments it needs, the currying employed under the hood to implement partial application in this case @seclink["Using_Racket_to_Define_Flows"]{evaluates to a function result right away}. This then receives the value @racket[3], and consequently, this expression produces the correct result.
-
 So in sum, it's perhaps best to rely on @racket[esc] in such cases to be as explicit as possible about what you mean, rather than rely on quirks of the implementation that are revealed at this boundary between two languages.
 
 @subsubsection{Mutable Values Defy the Laws of Flows}
@@ -359,6 +354,18 @@ Worse still, even though this computation raises an error, we find that the orig
 
 So in general, use mutable values with caution. Such values can be useful as side effects, for instance to capture some idea of statefulness, perhaps keeping track of the number of times a @tech{flow} was invoked. But they should generally not be used as inputs to a flow, especially if they are to be mutated.
 
+@subsubsection{Order of Effects}
+
+ Qi flows may exhibit a different order of effects (in the functional programming sense) than equivalent Racket functions.
+
+Consider the Racket expression: @racket[(map sqr (filter odd? (list 1 2 3 4 5)))]. As this invokes @racket[odd?] on all of the elements of the input list, followed by @racket[sqr] on all of the elements of the intermediate list, if we imagine that @racket[odd?] and @racket[sqr] print their inputs as a side effect before producing their results, then executing this program would print the numbers in the sequence @racket[1,2,3,4,5,1,3,5].
+
+The equivalent Qi flow is @racket[(~> ((list 1 2 3 4 5)) (filter odd?) (map sqr))]. As this sequence is @seclink["Don_t_Stop_Me_Now"]{"deforested" by Qi's compiler} to avoid multiple passes over the data and the memory overhead of intermediate representations, it invokes the functions in sequence @emph{on each element} rather than @emph{on all of the elements of each list in turn}. The printed sequence with Qi would be @racket[1,1,2,3,3,4,5,5].
+
+Yet, either implementation produces the same output: @racket[(list 1 9 25)].
+
+So, to reiterate, while the output of Qi flows will be the same as the output of equivalent Racket expressions, they may nevertheless exhibit a different order of effects.
+
 @section{Effectively Using Feedback Loops}
 
 @racket[feedback] is Qi's most powerful looping form, useful for arbitrary recursion. As it encourages quite a different way of thinking than Racket's usual looping forms do, here are some tips on "grokking" it.