split cli #586
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -210,7 +210,7 @@ const captures = await parsers.code(file, "(interface_declaration) @i") | |
| The `parsers.math` function uses [mathjs](https://mathjs.org/) to parse a math expression. | ||
|
|
||
| ```js | ||
| const res = await parsers.math("1 + 1") | ||
| ``` | ||
|
There was a problem hiding this comment. The
There was a problem hiding this comment. The
|
||
|
|
||
| ## .env | ||
|
There was a problem hiding this comment. Duplicate section 'math' in documentation.
|
||
|
|
@@ -272,14 +272,6 @@ for the current model. This is useful for estimating the number of prompts that | |
| const count = parsers.tokens("...") | ||
| ``` | ||
|
|
||
|
There was a problem hiding this comment. The section on
|
||
| ## validateJSON | ||
|
|
||
| The `parsers.validateJSON` function validates a JSON string against a schema. | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -548,7 +548,7 @@ defTool("math_eval", "Evaluates a math expression", { | |
| required: ["expression"], | ||
| }, async (args) => { | ||
| const { expression } = args | ||
|
There was a problem hiding this comment. The
|
||
| return "" + (await parsers.math(expression) ?? "?") | ||
|
There was a problem hiding this comment. The
There was a problem hiding this comment. The
|
||
| }) | ||
| ````` | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,82 +1,83 @@ | ||
| { | ||
| "name": "genaiscript", | ||
| "version": "1.45.1", | ||
| "main": "built/genaiscript.cjs", | ||
| "bin": { | ||
| "genaiscript": "./built/genaiscript.cjs" | ||
| }, | ||
| "publisher": "Microsoft", | ||
| "repository": { | ||
| "url": "https://github.com/microsoft/genaiscript" | ||
| }, | ||
| "homepage": "https://microsoft.github.io/genaiscript", | ||
| "keywords": [ | ||
| "genai", | ||
| "ai", | ||
| "cli", | ||
| "prompt", | ||
| "llm", | ||
| "generative ai", | ||
| "gpt4", | ||
| "chatgpt", | ||
| "ollama", | ||
| "llamacpp" | ||
| ], | ||
| "description": "A CLI for GenAIScript, a generative AI scripting framework.", | ||
| "license": "MIT", | ||
| "dependencies": { | ||
| "@azure/identity": "^4.4.0", | ||
| "@lvce-editor/ripgrep": "^1.2.0", | ||
| "dockerode": "^4.0.2", | ||
| "mammoth": "^1.8.0", | ||
| "mathjs": "^13.0.2", | ||
| "pdfjs-dist": "4.4.168", | ||
| "promptfoo": "^0.72.0", | ||
| "tree-sitter-wasms": "^0.1.11", | ||
| "tsx": "^4.16.2", | ||
| "typescript": "5.5.3", | ||
| "vectra": "^0.7.6", | ||
| "web-tree-sitter": "^0.22.2", | ||
| "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" | ||
| }, | ||
| "engines": { | ||
| "node": ">=20.0.0" | ||
| }, | ||
| "devDependencies": { | ||
| "@types/dockerode": "^3.3.30", | ||
| "@types/fs-extra": "^11.0.4", | ||
| "@types/memorystream": "^0.3.4", | ||
| "@types/node": "^20.14.11", | ||
| "@types/papaparse": "^5.3.14", | ||
| "@types/prompts": "^2.4.9", | ||
| "@types/replace-ext": "^2.0.2", | ||
| "@types/semver": "^7.5.8", | ||
| "@types/ws": "^8.5.11", | ||
| "commander": "^12.1.0", | ||
| "dotenv": "^16.4.5", | ||
| "esbuild": "^0.23.0", | ||
| "execa": "^9.3.0", | ||
| "file-type": "^19.1.1", | ||
| "fs-extra": "^11.2.0", | ||
| "get-stdin": "^9.0.0", | ||
| "glob": "^11.0.0", | ||
| "memorystream": "^0.3.1", | ||
| "node-sarif-builder": "^3.1.0", | ||
| "openai": "^4.52.7", | ||
| "ora": "^8.0.1", | ||
| "pretty-bytes": "^6.1.1", | ||
| "prompts": "^2.4.2", | ||
| "replace-ext": "^2.0.0", | ||
| "semver": "^7.6.3", | ||
| "zx": "^8.1.4" | ||
| }, | ||
| "scripts": { | ||
| "compile": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity&& node ../../scripts/patch-cli.mjs", | ||
| "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity", | ||
| "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md", | ||
| "vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html", | ||
| "vis:network": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.network.html --template network", | ||
| "go": "yarn compile && node built/genaiscript.cjs", | ||
| "test": "node --import tsx --test src/**.test.ts", | ||
| "typecheck": "tsc -p src" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| import { AZURE_OPENAI_TOKEN_SCOPES } from "../../core/src/constants" | ||
|
|
||
| export async function createAzureToken(signal: AbortSignal): Promise<string> { | ||
| const { DefaultAzureCredential } = await import("@azure/identity") | ||
| const azureToken = await new DefaultAzureCredential().getToken( | ||
| AZURE_OPENAI_TOKEN_SCOPES.slice(), | ||
| { abortSignal: signal } | ||
| ) | ||
| return azureToken.token | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,7 +10,6 @@ import { execa } from "execa" | |
| import { join } from "node:path" | ||
| import { createNodePath } from "./nodepath" | ||
| import { DockerManager } from "./docker" | ||
| import { createFileSystem } from "../../core/src/filesystem" | ||
| import { filterGitIgnore } from "../../core/src/gitignore" | ||
| import { | ||
|
|
@@ -44,6 +43,7 @@ import { createBundledParsers } from "../../core/src/pdf" | |
| import { AbortSignalOptions, TraceOptions } from "../../core/src/trace" | ||
| import { logVerbose, unique } from "../../core/src/util" | ||
| import { parseModelIdentifier } from "../../core/src/models" | ||
| import { createAzureToken } from "./azuretoken" | ||
|
|
||
| class NodeServerManager implements ServerManager { | ||
| async start(): Promise<void> { | ||
|
|
@@ -138,7 +138,7 @@ export class NodeHost implements RuntimeHost { | |
| await parseDefaultsFromEnv(process.env) | ||
| } | ||
|
|
||
| private _azureToken: string | ||
| async getLanguageModelConfiguration( | ||
| modelId: string, | ||
| options?: { token?: boolean } & AbortSignalOptions & TraceOptions | ||
|
|
@@ -152,14 +152,10 @@ export class NodeHost implements RuntimeHost { | |
| !tok.token && | ||
| tok.provider === MODEL_PROVIDER_AZURE | ||
| ) { | ||
| if (!this._azureToken) | ||
| this._azureToken = await createAzureToken(signal) | ||
|
There was a problem hiding this comment. There is no error handling for the
|
||
| if (!this._azureToken) throw new Error("Azure token not available") | ||
| tok.token = "Bearer " + this._azureToken | ||
| } | ||
| return tok | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,5 @@ | ||
| import { host } from "./host" | ||
| import { TraceOptions } from "./trace" | ||
|
|
||
| /** | ||
| * parses docx, require mammoth to be installed | ||
|
|
@@ -15,6 +13,7 @@ export async function DOCXTryParse( | |
| ): Promise<string> { | ||
| const { trace } = options || {} | ||
| try { | ||
| const { extractRawText } = await import("mammoth") | ||
|
There was a problem hiding this comment. Dynamic import of "mammoth" module is used. This could lead to performance issues as it might cause delays in execution. Consider importing the module at the top of the file if it does not have side effects. 😊
There was a problem hiding this comment. The
|
||
| const path = !/^\//.test(file) | ||
| ? host.path.join(host.projectFolder(), file) | ||
| : file | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
parsers.mathfunction should be used withawaitas it returns a Promise.