Fix #104: add Graal native image config and tests (#106)

* Add Graal native image config and tests

The clj-yaml jar now includes the necessary native-image config
for GraalVM's `native-image`. This config is picked up automatically by
GraalVM `native-image` and frees up native-image users from having to
figure out and specify this config.

We now have a babashka `test-native` task that verifies that clj-yaml
does in fact work when compiled with GraalVM `native-image`. We natively
compile our full test suite.

The native image is built against a locally built clj-yaml jar file
instead of local sources. This verifies that our native image config
within our jar is picked up.

* ci: yaml fix

.github/workflows/shared-setup/action.yml

@@ -28,6 +28,7 @@ runs:
       with:
         distribution: 'temurin'
         java-version: ${{ inputs.jdk }}
+      if: inputs.jdk != 'skip'
 
     - name: Install Clojure Tools
       uses: DeLaGuardo/[email protected]

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
         os: [{name: 'windows', shell: 'pwsh'}, {name: 'ubuntu', shell: 'bash'}]
         jdk: ['8', '11', '17']
 
-    name: ${{matrix.os.name}} - jdk ${{ matrix.jdk }}
+    name: ${{matrix.os.name}} jdk ${{ matrix.jdk }}
 
     steps:
     - name: Checkout
@@ -63,3 +63,38 @@ jobs:
 
     - name: Run tests
       run: bb test --clj-version :all
+
+  test-native:
+    runs-on: ${{matrix.os.name}}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [{name: 'windows', shell: 'pwsh'}, {name: 'ubuntu', shell: 'bash'}]
+        distribution:
+          - { name: 'graalvm', short-name: 'graal' }
+          - { name: 'graalvm-community', short-name: 'graalce' }
+        java-version:
+          - '17.0.8'
+          - '20.0.2'
+
+    name: ${{matrix.os.name}} ${{matrix.distribution.short-name}} ${{matrix.java-version}}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Setup
+      uses: ./.github/workflows/shared-setup
+      with:
+        jdk: skip
+        shell: ${{ matrix.os.shell }}
+
+    - name: Setup GraalVM
+      uses: graalvm/setup-graalvm@v1
+      with:
+        java-version: ${{ matrix.java-version }}
+        distribution: ${{ matrix.distribution.name }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run native tests
+      run: bb test-native

CHANGELOG.adoc

@@ -22,6 +22,9 @@ Clj-yaml makes use of SnakeYAML, please also refer to the https://bitbucket.org/
 ** Added `:code-point-limit` option to accept bigger documents
 (https://github.com/clj-commons/clj-yaml/issues/94[#94])
 (https://github.com/pitalig[@pitalig])
+** Added GraalVM native-image configuration (and tests)
+(https://github.com/clj-commons/clj-yaml/issues/104[#104])
+(https://github.com/lead[@lread])
 * Dependencies
 ** Bump `org.flatland/ordered` to `1.15.11`
 (https://github.com/clj-commons/clj-yaml/issues/98[#98])

bb.edn

@@ -13,8 +13,14 @@
          ;; tasks
          clean
          {:doc "clean build work"
-          :task (when (fs/exists? "target")
-                  (fs/delete-tree "target"))}
+          :task (do
+                  (println "Deleting (d=deleted -=did not exist)")
+                  (run! (fn [d]
+                          (println (format "[%s] %s"
+                                           (if (fs/exists? d) "d" "-")
+                                           d))
+                          (fs/delete-tree d))
+                        ["target" ".cpcache"]))}
          download-deps
          {:doc "bring down Clojure deps"
           :task download-deps/-main}
@@ -28,6 +34,9 @@
                   (when (not (fs/exists? "target/classes"))
                     (run 'compile-java))
                   (apply test-clj/-main *command-line-args*))}
+         test-native
+         {:doc "Run tests natively compiled (requires GraalVM)"
+          :task test-native/-main}
          lint-kondo
          {:doc "[--rebuild] Lint source code with clj-kondo"
           :task lint/-main}

build.clj

@@ -1,5 +1,6 @@
 (ns build
-  (:require [build-shared]
+  (:require [babashka.fs :as fs]
+            [build-shared]
             [clojure.java.shell :as shell]
             [clojure.tools.build.api :as b]))
 
@@ -8,6 +9,7 @@
 
 ;; build constants
 (def class-dir "target/classes")
+(def native-test-class-dir "target/native-test-classes") ;; keep this separate
 (def basis (b/create-basis {:project "deps.edn"}))
 (def jar-file (format "target/%s-%s.jar" (name lib) version))
 
@@ -32,7 +34,7 @@
     (println "compile-java with java version" version)
     (when (< major 8)
       (throw (ex-info "jdk version must be at least 8" {})))
-    (let [javac-opts ["-Xlint" "-Werror"]]
+    (let [javac-opts ["-Xlint:-options" "-Werror"]]
       (b/javac (cond-> {:src-dirs ["src/java"]
                         :class-dir class-dir
                         :basis basis
@@ -42,6 +44,27 @@
                  ;; --release replaces -source and -target opts for > jdk8
                  (update :javac-opts #(conj % "--release" "8")))))))
 
+
+(defn compile-clj-for-native-test
+  "We compile our tests against our local jar."
+  [_]
+  (println "compile-clj to:" native-test-class-dir)
+  (let [jars (->> (fs/glob "target" "*.jar") (mapv str))]
+    (when (not= (count jars) 1)
+      (throw (ex-info (format "Expected 1 jar under ./target to compile against, but found: %s"
+                              (if (seq jars) jars "none"))
+                      {})))
+    (let [jar (first jars)
+          basis (b/create-basis {:aliases [:native-test :test]
+                                 :extra {:deps {'clj-commons/clj-yaml {:local/root jar}}}})]
+      (println "Using jar:" jar)
+      ;; share the classpath for native-image to use in test-native bb task
+      (spit "target/native-classpath.edn" (pr-str (:classpath-roots basis)))
+      (b/compile-clj {:basis basis
+                      :class-dir native-test-class-dir
+                      :src-dirs ["test"]
+                      :ns-compile ['clj-yaml.native-test-runner]}))))
+
 (defn jar [_]
   (compile-java nil)
   (println "jarring version" version)
@@ -51,7 +74,7 @@
                 :scm {:tag (build-shared/version->tag version)}
                 :basis basis
                 :src-dirs ["src/clojure"]})
-  (b/copy-dir {:src-dirs ["src/clojure"]
+  (b/copy-dir {:src-dirs ["src/clojure" "resources"]
                :target-dir class-dir})
   (b/jar {:class-dir class-dir
           :jar-file jar-file}))

deps.edn

@@ -1,4 +1,4 @@
-{:paths ["src/clojure" "target/classes"]
+{:paths ["src/clojure" "target/classes" "resources"]
  :deps {org.yaml/snakeyaml {:mvn/version "2.1"}
         org.flatland/ordered {:mvn/version "1.15.11"}}
  :deps/prep-lib {:alias :build
@@ -17,10 +17,15 @@
          :extra-deps {io.github.cognitect-labs/test-runner
                       {:git/tag "v0.5.1" :git/sha "dfb30dd"}}
          :main-opts ["-m" "cognitect.test-runner"]}
+  :native-test
+  {:override-deps {org.clojure/clojure {:mvn/version "1.11.1"}}
+   :replace-paths ["target/native-test-classes"]
+   :extra-deps {com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"}}}
   :build
   {:extra-paths ["build"]
    :deps {io.github.clojure/tools.build {:mvn/version "0.9.4"}
-          slipset/deps-deploy {:mvn/version "0.2.1"}}
+          slipset/deps-deploy {:mvn/version "0.2.1"}
+          babashka/fs {:mvn/version "0.4.19"}}
    :ns-default build}
   ;; for consistent linting we use a specific version of clj-kondo through the jvm
   :clj-kondo {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.07.13"}}

doc/01-user-guide.adoc

@@ -322,3 +322,9 @@ Clojure 1.11 allows these types of functions to instead be called with a map for
 ----
 
 TIP: If you are using a version of Clojure before v1.11, or you want to stay compatible with older versions of Clojure, you'll need to call these functions the old school way.
+
+=== With GraalVM native-image
+
+Clj-yaml includes a GraalVM native image configuration so that it can compile without any external config.
+We run the clj-yaml test suite natively compiled by the current versions of GraalVM.
+Older versions of GraalVM are not supported.

doc/02-developer-guide.adoc

@@ -28,6 +28,7 @@ When we merge your PR, we'll usually squash it, so that will clean up any rambli
 
 The current version of Babashka.
 The current version of Clojure.
+The current version of GraalVM (to run native image tests)
 JDK8+
 Some knowledge of Java if you are going to add/modify Java code (there's not much!).
 
@@ -95,6 +96,21 @@ $ bb test --clj-version 1.9
 ----
 (defaults to `1.8`, specify `:all` to test against all supported Clojure versions)
 
+=== GraalVM testing
+
+With your dev environment configured to a current version of GraalVM run:
+[source,shell]
+----
+$ bb test-native
+----
+
+There is currently no facility to run a subest of tests.
+
+We include a link:/resources/META-INF/native-image/clj-commons/clj-yaml/native-image.properties[native-image config] in our jar file for GraalVM to pick up.
+To ensure that this works, we natively compile our test sources against a locally built clj-yaml jar.
+
+Our `clj-yaml.native-test-runner` is currently hand-coded, if you add any test namespaces you'll need to adjust it.
+
 === Linting
 Our CI workflow lints sources with clj-kondo, and eastwood - and you can too!
 

resources/META-INF/native-image/clj-commons/clj-yaml/native-image.properties

@@ -0,0 +1 @@
+Args = --initialize-at-build-time=org.yaml.snakeyaml

script/test_native.clj

@@ -0,0 +1,120 @@
+(ns test-native
+  (:require [lread.status-line :as status]
+            [babashka.fs :as fs]
+            [babashka.tasks :as t]
+            [cheshire.core :as json]
+            [clojure.edn :as edn]
+            [clojure.math :as math]
+            [clojure.string :as str]))
+
+(defn- find-graal-prog [prog-name]
+  (or (fs/which prog-name)
+      (fs/which (str (fs/file (System/getenv "JAVA_HOME") "bin")) prog-name)
+      (fs/which (str (fs/file (System/getenv "GRAALVM_HOME") "bin")) prog-name)))
+
+(defn- find-graal-native-image
+  "The Graal team now bundle native-image with Graal, there is no longer any need to install it."
+  []
+  (status/line :head "Locate GraalVM native-image")
+  (let [native-image (or (find-graal-prog "native-image")
+                         (status/die 1 "failed to to find GraalVM native-image, it should be bundle with your Graal installation"))]
+    (status/line :detail (str "found: " native-image))
+    native-image))
+
+(defn get-classpath []
+  (status/line :head "Get classpath")
+  (let [classpath (-> "target/native-classpath.edn"
+                      slurp
+                      edn/read-string)]
+
+    (println "\nClasspath:")
+    (println (str "- " (str/join "\n- " classpath)))
+    (str/join fs/path-separator classpath)))
+
+(defn generate-reflection-config [target-file]
+  ;; we add these classes to support our "allow unsafe" tests
+  (->> [{:name "javax.script.ScriptEngineManager"
+         :queryAllDeclaredConstructors true}
+        {:name "java.net.URLClassLoader"
+         :queryAllDeclaredConstructors true}
+        {:name "java.net.URL"
+         :queryAllDeclaredConstructors true
+         :methods [{:name "<init>" :parameterTypes ["java.lang.String"] }]}]
+       (json/generate-string)
+       (spit target-file)))
+
+(defn run-native-image [{:keys [:graal-native-image
+                                :reflection-config
+                                :target-path :target-exe :classpath :native-image-xmx
+                                :entry-class]}]
+  (status/line :head "Graal native-image compile AOT")
+  (let [full-target-exe (fs/which target-exe)]
+    (when (fs/exists? full-target-exe)
+      (fs/delete full-target-exe)))
+  (let [native-image-cmd [graal-native-image
+                          (str "-H:Path=" target-path)
+                          (str "-H:Name=" target-exe)
+                          "--features=clj_easy.graal_build_time.InitClojureClasses"
+                          "-O1" ;; basic optimization for faster build
+                          (str "-H:ReflectionConfigurationFiles=" reflection-config) ;; to support unsafe yaml test
+                          "-H:+ReportExceptionStackTraces"
+                          "--verbose"
+                          "--no-fallback"
+                          "-cp" classpath
+                          (str "-J-Xmx" native-image-xmx)
+                          entry-class]]
+    (t/shell native-image-cmd)))
+
+(defn humanize-bytes [bytes]
+  (let [units ["bytes" "KB" "MB" "GB"]
+        max-exponent (dec (count units))
+        base 1024
+        exponent (if (zero? bytes)
+                   0
+                   (int (math/floor (/ (math/log bytes) (math/log base)))))
+        exponent (if (> exponent max-exponent)
+                   max-exponent
+                   exponent)
+        in-bytes (format "%,d bytes" bytes)]
+    (if (zero? exponent)
+      in-bytes
+      (format "%.2f %s (%s)"
+              (/ bytes (math/pow base exponent))
+              (units exponent)
+              in-bytes))))
+
+(defn -main [& _args]
+  (let [native-image-xmx "6g"
+        target-path "target"
+        target-exe "clj-yaml-test"
+        graal-native-image (find-graal-native-image)
+        reflection-config "target/reflect-config.json"]
+    (status/line :head "Creating native image for test")
+    (status/line :detail "java -version")
+    (t/shell "java -version")
+    (status/line :detail (str "\nnative-image max memory: " native-image-xmx))
+    (fs/create-dirs target-path)
+    (status/line :head "Creating clj-yaml jar to test against")
+    (t/clojure "-T:build jar")
+    (status/line :head "Generating reflection config to support unsafe tests")
+    (generate-reflection-config reflection-config)
+    (status/line :head "AOT Compiling test sources")
+    (t/clojure "-T:build compile-clj-for-native-test")
+    (let [classpath (get-classpath)]
+      (run-native-image {:graal-native-image graal-native-image
+                         :reflection-config reflection-config
+                         :target-path target-path
+                         :target-exe target-exe
+                         :classpath classpath
+                         :native-image-xmx native-image-xmx
+                         :entry-class "clj_yaml.native_test_runner"}))
+    (status/line :head "Native image built")
+    (let [full-target-exe (fs/which (fs/file target-path target-exe))]
+      (status/line :detail "built: %s, %s" full-target-exe (humanize-bytes (fs/size full-target-exe)))
+      (status/line :head "Running tests natively")
+      (t/shell full-target-exe)))
+  nil)
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
+

test/clj_yaml/native_test_runner.clj

@@ -0,0 +1,24 @@
+(ns clj-yaml.native-test-runner
+  "Test runner used for Graal native image tests.
+  Namespace cannot be automatically discovered during a native image test,
+  so we specify them explicitly so that they will be compiled in.
+  Any new test namespaces will need to be manually added."
+ (:gen-class)
+ (:require
+  [clojure.test :as t]
+  [clj-yaml.core-test]))
+
+(defn
+ -main
+ [& _args]
+ (println "clojure version" (clojure-version))
+ (println "java version" (System/getProperty "java.version"))
+ (println
+  "running native?"
+  (= "executable" (System/getProperty "org.graalvm.nativeimage.kind")))
+ (let
+  [{:keys [fail error]}
+   (apply
+    t/run-tests
+    '(clj-yaml.core-test))]
+  (System/exit (if (zero? (+ fail error)) 0 1))))