transformers.js support

.vscode/settings.json

@@ -29,7 +29,8 @@
         "treesitter",
         "typecheck",
         "vsix",
-        "xpai"
+        "xpai",
+        "xenova"
     ],
     "sarif-viewer.connectToGithubCodeScanning": "on",
     "cSpell.language": "en,en-US",

cspell.config.yaml

@@ -0,0 +1,9 @@
+version: "0.2"
+ignorePaths:
+  - yarn.lock
+dictionaryDefinitions: []
+dictionaries: []
+words:
+  - testhost
+ignoreWords: []
+import: []

packages/cli/package.json

@@ -23,6 +23,7 @@
   "description": "A CLI for GenAIScript, a generative AI scripting framework.",
   "license": "MIT",
   "dependencies": {
+    "@xenova/transformers": "^2.17.1",
     "@azure/identity": "^4.3.0",
     "dockerode": "^4.0.2",
     "pdfjs-dist": "4.3.136",

packages/core/package.json

@@ -12,6 +12,7 @@
     "url": "https://github.com/microsoft/genaiscript"
   },
   "optionalDependencies": {
+    "@xenova/transformers": "^2.17.1",
     "pdfjs-dist": "4.3.136",
     "web-tree-sitter": "^0.22.2"
   },

packages/core/src/testhost.ts

@@ -45,10 +45,10 @@ export class TestHost implements Host {
         return resolve(".")
     }
     installFolder(): string {
-        throw new Error("Method not implemented.")
+        return this.projectFolder()
     }
-    resolvePath(...segments: string[]): string {
-        throw new Error("Method not implemented.")
+    resolvePath(...segments: string[]) {
+        return resolve(...segments)
     }
     readSecret(name: string): Promise<string> {
         throw new Error("Method not implemented.")

packages/core/src/transformers.test.ts

@@ -0,0 +1,16 @@
+import { sentiment } from "./transformers"
+import { readFile } from "fs/promises"
+import { beforeEach, describe, test } from "node:test"
+import assert from "node:assert/strict"
+import { TestHost } from "./testhost"
+
+describe("sentiment function", () => {
+    beforeEach(() => {
+        TestHost.install()
+    })
+    test("should process love", async () => {
+        const result = await sentiment("i love u")
+        assert.strictEqual(result.label, "POSITIVE")
+        assert(result.score > 0.9)
+    })
+})

packages/core/src/transformers.ts

@@ -0,0 +1,13 @@
+import { TextClassificationOutput, pipeline } from "@xenova/transformers"
+import { dotGenaiscriptPath } from "./util"
+
+export async function sentiment(text: string) {
+    const cache_dir = dotGenaiscriptPath("cache", "transformers")
+    console.log(cache_dir)
+    const pipe = await pipeline("sentiment-analysis", undefined, {
+        cache_dir,
+    })
+    const out = (await pipe(text)) as TextClassificationOutput
+    console.log(out)
+    return out[0]
+}