Update transformers.js to use Hugging Face library (#880)

* refresh transformers.js

* dont' force changelog with prediciotn

* refactor: ♻️ remove max_new_tokens from generator call

.gitignore

@@ -27,3 +27,4 @@ trapi.md
 .env.local
 genaiscript.config.yaml
 packages/core/*.temp.*
+*.vsix

THIRD_PARTY_LICENSES.md

@@ -1464,7 +1464,7 @@ available under the Apache 2.0 license (https://github.com/puppeteer/puppeteer/b
 
 The following npm package may be included in this product:
 
- - [email protected].2
+ - [email protected].3
 
 This package contains the following license:
 
@@ -1670,7 +1670,7 @@ The following npm packages may be included in this product:
 
  - @types/http-cache-semantics@4.0.4
  - @types/node@16.9.1
- - @types/node@22.9.0
+ - @types/node@22.9.1
  - @types/turndown@5.0.5
  - @types/yauzl@2.10.3
 
@@ -1702,7 +1702,7 @@ MIT License
 
 The following npm package may be included in this product:
 
- - [email protected].1
+ - [email protected].2
 
 This package contains the following license:
 
@@ -4862,9 +4862,9 @@ The following npm packages may be included in this product:
  - @tokenizer/token@0.3.0
  - [email protected]
  - [email protected]
- - [email protected].1
- - [email protected].1
- - [email protected].1
+ - [email protected].2
+ - [email protected].2
+ - [email protected].2
  - [email protected]
  - [email protected]
  - [email protected]
@@ -5546,7 +5546,7 @@ SOFTWARE.
 
 The following npm package may be included in this product:
 
- - @azure/msal-node@2.16.1
+ - @azure/msal-node@2.16.2
 
 This package contains the following license:
 

docs/src/content/docs/guides/transformers-js.mdx

@@ -28,16 +28,16 @@ for more.
 ## Installation
 
 Following the [installation instructions](https://huggingface.co/docs/transformers.js/installation), 
-we add the [@xenova/transformers](https://www.npmjs.com/package/@xenova/transformers) to the current project.
+we add the [@huggingface/transformers](https://www.npmjs.com/package/@huggingface/transformers) to the current project.
 
 ```bash
-npm install @xenova/transformers
+npm install @huggingface/transformers
 ```
 
 You can also install this library globally to be able to use on any project
 
 ```bash "-g"
-npm install -g @xenova/transformers
+npm install -g @huggingface/transformers
 ```
 
 ## Import the pipeline
@@ -46,7 +46,7 @@ The snippet below imports the Transformers.js library and loads the summarizer p
 You can specify a model name or let the library pick the latest and greatest.
 
 ```js
-import { pipeline } from "@xenova/transformers"
+import { pipeline } from "@huggingface/transformers"
 const summarizer = await pipeline("summarization")
 ```
 

docs/yarn.lock

@@ -136,9 +136,9 @@
     zod "^3.23.8"
 
 "@astrojs/starlight@^0.29.0":
-  version "0.29.1"
-  resolved "https://registry.yarnpkg.com/@astrojs/starlight/-/starlight-0.29.1.tgz#e13f68926ebe3f7f10c3b32e415140cac82534b0"
-  integrity sha512-0f/YRTo9oche2AqPyPzhKMl5uT09ANM8Yuwb54AIaWOnOYXsedEekS+ryYVsz5jo3F0AfA7MlZhwqj+z2ZtmcA==
+  version "0.29.2"
+  resolved "https://registry.yarnpkg.com/@astrojs/starlight/-/starlight-0.29.2.tgz#157b1edf8dcd7e6d13905e89b2cbe3ee1d745f7d"
+  integrity sha512-xv9AhWkP3fxCB6EF6MlT4yEbxzye3aMSbuVbFEGbQh8G/w1MPhdNCnQakIHpmIwwyxwG9cW3mQdAZum4oOO39w==
   dependencies:
     "@astrojs/mdx" "^3.1.3"
     "@astrojs/sitemap" "^3.1.6"
@@ -601,9 +601,9 @@
     "@expressive-code/core" "^0.38.3"
 
 "@fortawesome/fontawesome-free@^6.0.0":
-  version "6.7.0"
-  resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-6.7.0.tgz#c38f4c32970628e214626697edae34a7fe282dbd"
-  integrity sha512-lXTP8oiLGM97Aqjn47nod9awKe574HRfCngFizr7FiHTadKTiPkoVn0Mzg+tZBLD1XpE4CRvcF7/UM3g1jyVYw==
+  version "6.7.1"
+  resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-6.7.1.tgz#160a48730d533ec77578ed0141661a8f0150a71d"
+  integrity sha512-ALIk/MOh5gYe1TG/ieS5mVUsk7VUIJTJKPMK9rFFqOgfp0Q3d5QiBXbcOMwUvs37fyZVCz46YjOE6IFeOAXCHA==
 
 "@iconify/types@^2.0.0":
   version "2.0.0"
@@ -1265,11 +1265,11 @@
     "@types/ms" "*"
 
 "@types/dompurify@^3.0.5":
-  version "3.0.5"
-  resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-3.0.5.tgz#02069a2fcb89a163bacf1a788f73cb415dd75cb7"
-  integrity sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-3.2.0.tgz#56610bf3e4250df57744d61fbd95422e07dfb840"
+  integrity sha512-Fgg31wv9QbLDA0SpTOXO3MaxySc4DKGLi8sna4/Utjo4r3ZRPdCt4UQee8BWr+Q5z21yifghREPJGYaEOEIACg==
   dependencies:
-    "@types/trusted-types" "*"
+    dompurify "*"
 
 "@types/estree-jsx@^1.0.0":
   version "1.0.5"
@@ -1335,9 +1335,9 @@
     "@types/unist" "*"
 
 "@types/node@*", "@types/node@>=20":
-  version "22.9.0"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.9.0.tgz#b7f16e5c3384788542c72dc3d561a7ceae2c0365"
-  integrity sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==
+  version "22.9.1"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-22.9.1.tgz#bdf91c36e0e7ecfb7257b2d75bf1b206b308ca71"
+  integrity sha512-p8Yy/8sw1caA8CdRIQBG5tiLHmxtQKObCijiAa9Ez+d4+PRffM4054xbju0msf+cvhJpnFEeNjxmVT/0ipktrg==
   dependencies:
     undici-types "~6.19.8"
 
@@ -1353,7 +1353,7 @@
   dependencies:
     "@types/node" "*"
 
-"@types/trusted-types@*":
+"@types/trusted-types@^2.0.7":
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.7.tgz#baccb07a970b91707df3a3e8ba6896c57ead2d11"
   integrity sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==
@@ -1431,9 +1431,9 @@
     vscode-uri "^3.0.8"
 
 "@vscode/emmet-helper@^2.9.3":
-  version "2.10.0"
-  resolved "https://registry.yarnpkg.com/@vscode/emmet-helper/-/emmet-helper-2.10.0.tgz#a0add01dddce3e07b3f4169b559067096203263a"
-  integrity sha512-UHw1EQRgLbSYkyB73/7wR/IzV6zTBnbzEHuuU4Z6b95HKf2lmeTdGwBIwspWBSRrnIA1TI2x2tetBym6ErA7Gw==
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/@vscode/emmet-helper/-/emmet-helper-2.11.0.tgz#7a53e4fdb17329cc2ed88036905c78d811d231d6"
+  integrity sha512-QLxjQR3imPZPQltfbWRnHU6JecWTF1QSWhx3GAKQpslx7y3Dp6sIIXhKjiUJ/BR9FX8PVthjr9PD6pNwOJfAzw==
   dependencies:
     emmet "^2.4.3"
     jsonc-parser "^2.3.0"
@@ -2242,6 +2242,13 @@ dlv@^1.1.3:
   resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79"
   integrity sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==
 
+dompurify@*:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.2.1.tgz#d480972aeb1a59eb8ac47cba95558fbd72a0127b"
+  integrity sha512-NBHEsc0/kzRYQd+AY6HR6B/IgsqzBABrqJbpCDQII/OK6h7B7LXzweZTDsqSW2LkTRpoxf18YUP+YjGySk6B3w==
+  optionalDependencies:
+    "@types/trusted-types" "^2.0.7"
+
 "dompurify@^3.0.11 <3.1.7":
   version "3.1.6"
   resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.1.6.tgz#43c714a94c6a7b8801850f82e756685300a027e2"
@@ -2850,9 +2857,9 @@ http-cache-semantics@^4.1.1:
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
 
 i18next@^23.11.5:
-  version "23.16.5"
-  resolved "https://registry.yarnpkg.com/i18next/-/i18next-23.16.5.tgz#53d48ae9f985fd73fc1fcb96e6c7d90ababf0831"
-  integrity sha512-KTlhE3EP9x6pPTAW7dy0WKIhoCpfOGhRQlO+jttQLgzVaoOjWwBWramu7Pp0i+8wDNduuzXfe3kkVbzrKyrbTA==
+  version "23.16.8"
+  resolved "https://registry.yarnpkg.com/i18next/-/i18next-23.16.8.tgz#3ae1373d344c2393f465556f394aba5a9233b93a"
+  integrity sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==
   dependencies:
     "@babel/runtime" "^7.23.2"
 

packages/core/src/promptdom.ts

@@ -274,7 +274,9 @@ function renderDefNode(def: PromptDefNode): string {
         dfence += "`"
     }
     const diffFormat =
-        body.length > 500 ? " preferred_output_format=CHANGELOG " : ""
+        body.length > 500 && !prediction
+            ? " preferred_output_format=CHANGELOG "
+            : ""
     const res =
         "\n" +
         (name ? name + ":\n" : "") +

packages/sample/genaisrc/summary-with-transformers.genai.mjs

@@ -1,6 +1,6 @@
 script({
     title: "summary of summary - transformers.js",
-    model: "ollama:phi3",
+    model: "ollama:llama3.2:1b",
     files: ["src/rag/markdown.md"],
     tests: {
         files: ["src/rag/markdown.md"],
@@ -9,7 +9,7 @@ script({
 })
 
 console.log("loading summarizer transformer")
-import { pipeline } from "@xenova/transformers"
+import { pipeline } from "@huggingface/transformers"
 const summarizer = await pipeline("summarization")
 
 for (const file of env.files) {

packages/sample/genaisrc/transformers-all.genai.mjs

@@ -2,7 +2,7 @@ script({
     files: [],
 })
 
-import { pipeline, env as transformersEnv } from "@xenova/transformers"
+import { pipeline, env as transformersEnv } from "@huggingface/transformers"
 
 // https://huggingface.co/docs/transformers.js/tutorials/node#model-caching
 transformersEnv.cacheDir = "./.genaiscript/cache/transformers"
@@ -31,7 +31,7 @@ const log = (res) => console.log(res)
         "Xenova/toxic-bert",
         { progress_callback }
     )
-    const output = await classifier("I hate you!", { topk: null })
+    const output = await classifier("I hate you!")
     log(output)
 }
 
@@ -66,9 +66,7 @@ const log = (res) => console.log(res)
         "the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the " +
         "Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second " +
         "tallest free-standing structure in France after the Millau Viaduct."
-    const output = await generator(text, {
-        max_new_tokens: 100,
-    })
+    const output = await generator(text)
     log(output)
 }
 

packages/sample/genaisrc/transformers-rank.genai.mts

@@ -2,7 +2,7 @@ script({
     files: "src/rag/*"
 })
 
-import { pipeline, cos_sim } from '@xenova/transformers';
+import { pipeline, cos_sim } from '@huggingface/transformers';
 
 const featureExtractor = await pipeline('feature-extraction');
 

packages/sample/package.json

@@ -20,8 +20,8 @@
     "@agentic/weather": "^7.2.0",
     "@azure/identity": "^4.5.0",
     "@azure/storage-blob": "^12.25.0",
+    "@huggingface/transformers": "^3.0.2",
     "@tidyjs/tidy": "^2.5.2",
-    "@xenova/transformers": "^2.17.2",
     "p-all": "^5.0.0",
     "vectorstore": "^0.0.4",
     "zod": "^3.23.8",