Merge pull request #66 from behrica/upgradeSciclojMLDeps

Upgrade scicloj ml deps
scicloj · Nov 30, 2024 · afaca69 · afaca69
2 parents 671ab0e + 3591e38
commit afaca69
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 54 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -31,5 +31,6 @@
 	"updateContentCommand": {"install python packages": "mkdir -p .venv && poetry install --sync" ,
 	                     "add link to python executable": "sudo ln -s /usr/bin/python3 /usr/local/bin/python"
                          },
-   "postStartCommand": "clojure -Sthreads 1 -P ; clojure -X:deps prep"		
+   "onCreateCommand": "clojure -Sthreads 1 -A:model-integration-tests:test:dev  -P"		
+
 }
diff --git a/build/gen_tests.clj b/build/gen_tests.clj
@@ -12,6 +12,5 @@
     #(let [p (.getAbsolutePath %)]
        (println :generate-tests p)
        (clay/make! {:source-path p
-                    :show false}))))
-  (shutdown-agents))
+                    :show false})))))
 
diff --git a/deps.edn b/deps.edn
@@ -7,12 +7,15 @@
         generateme/fastmath                       {:mvn/version "3.0.0-alpha3"}
         aerial.hanami/aerial.hanami               {:mvn/version "0.20.1"}
         org.scicloj/tableplot                     {:mvn/version "1-alpha14.1"}
-        org.scicloj/metamorph.ml                  {:mvn/version "0.10.3"}
         org.scicloj/scicloj.ml.smile              {:mvn/version "7.4.3"}
+        org.scicloj/metamorph.ml                  {:mvn/version "0.10.4"}
         org.scicloj/sklearn-clj                   {:mvn/version "0.4.1"}
         org.scicloj/scicloj.ml.xgboost            {:mvn/version "6.2.0"}
 
-        org.scicloj/scicloj.ml.tribuo             {:mvn/version "0.1.3"}
+        org.scicloj/scicloj.ml.tribuo             {:mvn/version "0.1.4"}
+        techascent/tech.ml.dataset                {:mvn/version "7.032"}   ;; no JVM crash
+        ;;techascent/tech.ml.dataset                {:mvn/version "7.033"}  ;; JVM crash
+        ;;techascent/tech.ml.dataset                {:mvn/version "7.034"} ;; JVM crash
         org.tribuo/tribuo-regression-sgd          {:mvn/version "4.3.1"}
         org.tribuo/tribuo-regression-tree         {:mvn/version "4.3.1"}
         org.tribuo/tribuo-regression-xgboost      {:mvn/version "4.3.1"}

diff --git a/model-integration-tests/model_integration_test.clj b/model-integration-tests/model_integration_test.clj
@@ -273,12 +273,9 @@ warnings.simplefilter('ignore')")
      #(verify-fn % iris)
 
      (-> model-specs
-         ;;https://github.com/scicloj/scicloj.ml.tribuo/issues/6 
-         (remove-model-type  :scicloj.ml.tribuo/classification)
          ;;https://github.com/scicloj/scicloj.ml.smile/issues/19
          (remove-model-type  :smile.classification/mlp)
-         ;;https://github.com/scicloj/scicloj.ml.xgboost/issues/1
-         (remove-model-type  :xgboost/classification)))))
+         ))))
 
 (deftest verify-classification-iris-nil-catmap-float
   (let [iris
@@ -291,32 +288,12 @@ warnings.simplefilter('ignore')")
 
      #(verify-fn % iris)
      (-> model-specs
-         ;;https://github.com/scicloj/scicloj.ml.tribuo/issues/6 
-         (remove-model-type  :scicloj.ml.tribuo/classification)
          ;;https://github.com/scicloj/scicloj.ml.smile/issues/19
          (remove-model-type  :smile.classification/mlp)
          ;;https://github.com/scicloj/scicloj.ml.xgboost/issues/1
          (remove-model-type  :xgboost/classification)
          ))))
 
-  ;;(classify (-> smile-model-specs first second) iris-3)
-
-(comment
-  (def iris-no-cats
-    (->
-     (data/iris-ds)
-     (ds/shuffle)
-     ;(ds-cat/reverse-map-categorical-xforms)
-     ;( ds/categorical->number) [:species] {} :float64)
-     ;(ds/assoc-metadata [:species] :categorical-map nil)
-     ))
-
-
-
-  (def model (ml/train iris-no-cats {:model-type :xgboost/classification
-                                     :num-class 4}))
-  (ml/predict iris-no-cats model)
-  )
 
 
 (comment

diff --git a/notebooks/noj_book/ml_basic.clj b/notebooks/noj_book/ml_basic.clj
@@ -14,7 +14,10 @@
             [scicloj.metamorph.ml.toydata :as data]
             [tech.v3.dataset :as ds]
             [scicloj.kindly.v4.kind :as kind]
-            [scicloj.kindly.v4.api :as kindly]))
+            [scicloj.kindly.v4.api :as kindly]
+            [tech.v3.dataset.categorical :as ds-cat]))
+
+
 
 ^:kindly/hide-code
 (require '[same.core :as same]
@@ -213,7 +216,7 @@ split
 
 (loss/classification-accuracy
  (:survived (ds-cat/reverse-map-categorical-xforms (:test split)))
- (:survived  lreg-prediction))
+ (:survived (ds-cat/reverse-map-categorical-xforms lreg-prediction)))
 
 (kindly/check = 0.7373737373737373)
 ;; Its performance is  better, 73 %
@@ -241,11 +244,11 @@ split
     (tc/rows))
 
 (kindly/check same/ish?
-              [["no" 0.64 0.35]
-               ["no" 0.57 0.42]
-               ["no" 0.85 0.14]
-               ["no" 0.88 0.11]
-               ["no" 0.88 0.11]])
+              [[0.0 0.64 0.35]
+               [0.0 0.57 0.42]
+               [0.0 0.85 0.14]
+               [0.0 0.88 0.11]
+               [0.0 0.88 0.11]])
 
 
 
@@ -271,3 +274,6 @@ split
 ;; So far we used a single split into 'train' and 'test' data, so we only get
 ;; a point estimate of the accuracy. This should be made more robust
 ;; via cross-validations and using different splits of the data.
+
+
+
diff --git a/notebooks/noj_book/render_tools.clj b/notebooks/noj_book/render_tools.clj
@@ -10,9 +10,7 @@
    [scicloj.metamorph.ml.preprocessing :as preprocessing]
    [tablecloth.api :as tc]
    [tech.v3.dataset.modelling :as ds-mod]
-   [tech.v3.datatype.functional :as dtf]
-   [libpython-clj2.python :as py])
-  )
+   [tech.v3.datatype.functional :as dtf]))
 
 
 (defn anchor-or-nothing [x text]
@@ -79,7 +77,6 @@
   ( [prefix] (render-key-info prefix {:level "##"
                                       :remove-s ""})))
 
-^:kindly/hide-code
 (defn kroki [s type format]
   (client/post "https://kroki.io/" {:content-type :json
                                     :as :byte-array

diff --git a/notebooks/noj_book/render_tools_sklearn.clj b/notebooks/noj_book/render_tools_sklearn.clj
@@ -1,18 +1,8 @@
-
 (ns noj-book.render-tools-sklearn
   (:require
-   [clj-http.client :as client]
-   [clojure.string :as str]
-   [clojure.walk :as walk]
+   [libpython-clj2.python :as py]
    [scicloj.kindly.v4.kind :as kind]
-   [scicloj.metamorph.core :as mm]
-   [scicloj.metamorph.ml :as ml]
-   [tablecloth.pipeline :as tc-mm]
-   [scicloj.metamorph.ml.preprocessing :as preprocessing]
-   [tablecloth.api :as tc]
-   [tech.v3.dataset.modelling :as ds-mod]
-   [tech.v3.datatype.functional :as dtf]
-   [libpython-clj2.python :as py]))
+   [scicloj.metamorph.ml :as ml]))
 
 (py/initialize!)
 (def doc->markdown (py/import-module "docstring_to_markdown"))

diff --git a/notebooks/noj_book/underlying_libraries.clj b/notebooks/noj_book/underlying_libraries.clj
@@ -1,7 +1,11 @@
 ;; # Underlying libraries
 
 ^:kindly/hide-code
-(ns noj-book.underlying-libraries)
+(ns noj-book.underlying-libraries 
+  (:require
+   [clojure.edn :as edn]
+   [scicloj.kindly.v4.kind :as kind]
+   [tablecloth.api :as tc]))
 
 ;; Noj consists of the following libraries:
 
@@ -23,7 +27,7 @@
 ;; * [same-ish](https://github.com/microsoft/same-ish) - approximate comparisons - useful for notebook testability
 
 ;; ## Machine learning
-;; * [metamorph.ml](https://github.com/scicloj/metamorph.ml) - machine learning platform [(reference)](https://cljdoc.org/badge/scicloj/metamorph.ml)
+;; * [metamorph.ml](https://github.com/scicloj/metamorph.ml) - machine learning platform [(reference)](https://cljdoc.org/d/scicloj/metamorph.ml)
 ;; * [scicloj.ml.tribuo](https://github.com/scicloj/scicloj.ml.tribuo) - [Tribuo](https://tribuo.org/) machine learning models
 ;; * [scicloj.ml.smile](https://github.com/scicloj/scicloj.ml.smile) - [Smile](https://haifengl.github.io/) (v 2.6) machine learning models
 ;; * [sklearn-clj](https://github.com/scicloj/sklearn-clj) - Plugin to use [sklearn](https://scikit-learn.org/) models in metamorph.ml
@@ -38,3 +42,38 @@
 ;; * [libpython-clj](https://github.com/clj-python/libpython-clj) - Python bindings [(reference)](https://clj-python.github.io/libpython-clj/)
 ;; * [kind-pyplot](https://github.com/scicloj/kind-pyplot) - Python plotting [(reference)](https://scicloj.github.io/kind-pyplot/)
 ;; * [ClojisR](https://github.com/scicloj/clojisr) - R bindings [(reference)](https://clj-python.github.io/libpython-clj/)
+^:kindly/hide-code
+(def direct-deps
+  (->
+   (edn/read-string (slurp "deps.edn"))
+   :deps
+   keys
+   ))
+^:kindly/hide-code
+(def all-deps-info
+  (-> 
+   (clojure.java.shell/sh  "clj" "-X:deps" "list" ":format" ":edn")
+   :out
+   (edn/read-string)))
+
+^:kindly/hide-code
+(def direct-deps-info 
+  (select-keys
+   all-deps-info
+   direct-deps))
+
+;; ## List of all direct noj dependencies 
+
+^:kindly/hide-code
+(kind/table
+ (->
+  (map
+   (fn [[dep info]]
+     (hash-map :lib (str dep)
+               :version (:mvn/version info)
+               :license (-> info :license :name))
+     )
+   direct-deps-info)
+  (tc/dataset)
+  (tc/select-columns [:lib :version :license])
+  (tc/order-by [:lib])))