diff --git a/.gitignore b/.gitignore index 6c531bd367..40934fdd8e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ packages/sample/*.slides.md vscode-extension-samples/ .DS_Store .genaiscript/temp -.genaiscript/retreival +.genaiscript/retrieval results/ .genaiscript/ applications/EdgePeeringAI \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 5de217f75b..28766a8f22 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "type": "node", "cwd": "${workspaceFolder}", "preLaunchTask": "npm: compile-cli", - "args": ["parse", "code", "packages/sample/src/tla/EWD998PCal.tla", "(interface_declaration) @i"] + "args": ["run", "packages/sample/genaisrc/summarize-import.genai.mjs", "packages/sample/src/questions.md"] }, { "name": "Run - sample", diff --git a/.vscode/settings.json b/.vscode/settings.json index ae05a2c8cc..e9ecf06890 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,9 @@ { "devicescript.devtools.autoStart": false, "cSpell.words": [ + "AICI", "genai", + "Genaiscript", "gpspec", "gpspecs", "gptool", diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 150e470e3f..2fd5f1d966 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -782,7 +782,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -913,7 +913,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1002,9 +1002,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/docs/src/content/docs/getting-started/installation.mdx b/docs/src/content/docs/getting-started/installation.mdx index 0a0a308a8f..9b8ccd56dd 100644 --- a/docs/src/content/docs/getting-started/installation.mdx +++ b/docs/src/content/docs/getting-started/installation.mdx @@ -81,7 +81,7 @@ to bug fixes earlier than the marketplace release. - ... - .genaiscript/ folder created by the extension to store supporting files - cache/ various cache files - - retreival/ retreival database caches + - retrieval/ retrieval database caches - ... supporting files - **genaiscript.vsix** diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx index a67e73c67d..7870cdfc8a 100644 --- a/docs/src/content/docs/index.mdx +++ b/docs/src/content/docs/index.mdx @@ -133,7 +133,7 @@ const { pages } = await parsers.PDF(env.files[0]) ```js wrap // embedding vector index and search -const { files } = await retreival.search("cats", env.files) +const { files } = await retrieval.search("cats", env.files) ``` diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index 27d1091fdc..5fead64872 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -94,10 +94,10 @@ Options: -h, --help display help for command ``` -## `retreival` +## `retrieval` ``` -Usage: genaiscript retreival [options] [command] +Usage: genaiscript retrieval [options] [command] RAG support @@ -111,10 +111,10 @@ Commands: help [command] display help for command ``` -### `retreival index` +### `retrieval index` ``` -Usage: genaiscript retreival index [options] +Usage: genaiscript retrieval index [options] Index a set of documents @@ -131,10 +131,10 @@ Options: -h, --help display help for command ``` -### `retreival search` +### `retrieval search` ``` -Usage: genaiscript retreival search [options] [files...] +Usage: genaiscript retrieval search [options] [files...] Search index @@ -145,10 +145,10 @@ Options: -h, --help display help for command ``` -### `retreival clear` +### `retrieval clear` ``` -Usage: genaiscript retreival clear [options] +Usage: genaiscript retrieval clear [options] Clear index to force re-indexing diff --git a/docs/src/content/docs/reference/scripts/embeddings-search.md b/docs/src/content/docs/reference/scripts/embeddings-search.md index 733b428311..adb1b3a835 100644 --- a/docs/src/content/docs/reference/scripts/embeddings-search.md +++ b/docs/src/content/docs/reference/scripts/embeddings-search.md @@ -9,7 +9,7 @@ keywords: embeddings search, similarity search, vector database, indexing, LLM A The `retrieval.search` indexes the input files using [embeddings](https://platform.openai.com/docs/guides/embeddings) into a vector database that can be used for similarity search. This is commonly referred to as Retrieval Augmented Generation (RAG). ```js -const { files, fragments } = await retreival.search("keyword", env.files) +const { files, fragments } = await retrieval.search("keyword", env.files) ``` The returned `files` object contains the file with @@ -18,7 +18,7 @@ concatenated embeddings, and the `fragments` object contains each individual fil You can use the result of `files` in the `def` function. ```js -const { files } = await retreival.search("keyword", env.files) +const { files } = await retrieval.search("keyword", env.files) def("FILE", files) ``` diff --git a/docs/src/content/docs/reference/scripts/functions.md b/docs/src/content/docs/reference/scripts/functions.md index 26b1df484d..7b082688c9 100644 --- a/docs/src/content/docs/reference/scripts/functions.md +++ b/docs/src/content/docs/reference/scripts/functions.md @@ -93,7 +93,7 @@ defFunction("current_weather", ...) Let's illustrate how functions come together with a question answering script. In the script below, we add the ``system.web_search` which registers the `web_search` function. This function -will call into `retreival.webSearch` as needed. +will call into `retrieval.webSearch` as needed. ```js file="answers.genai.js" script({ diff --git a/docs/src/content/docs/reference/scripts/imports.mdx b/docs/src/content/docs/reference/scripts/imports.mdx new file mode 100644 index 0000000000..ec8d8ad924 --- /dev/null +++ b/docs/src/content/docs/reference/scripts/imports.mdx @@ -0,0 +1,67 @@ +--- +title: Imports +sidebar: + order: 20 +--- + +import { Steps } from "@astrojs/starlight/components" +import { FileTree } from "@astrojs/starlight/components" + +By default, the scripts cannot import modules (they are evaled) and are expected to execute the code directly. + +You can add support for imports by following these steps. + +## Converting to a module + + + +
    + +
  1. + +Rename the `.genai.js` file +to [module file](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules#aside_%E2%80%94_.mjs_versus_.js) `.genai.mjs`. + + + +- genaisrc/ + - **poem.genai.mjs** // .js -> .mjs + + + +
  2. + +
  3. + +Wrap the script code in a function and make it the default export. You can leave `script` in the main scope. + +```js title="poem.genai.mjs" "export default async function() {" "}" +script(...) +export default async function() { + $`Write a poem.` +} +``` + +
  4. + +
+ +
+ +## Imports + +Once the file is converted, you can use static or dynamic imports as any other module file. + +```js +import { parse } from "ini" +... +export default async function () { + // static import + const res = parse("x = 1\ny = 2") + console.log(res) + + // dynamic import + const { stringify } = await import("ini") + console.log(stringify(res)) +} +``` diff --git a/docs/src/content/docs/reference/scripts/retreival.md b/docs/src/content/docs/reference/scripts/retreival.md index efe0e0d4b8..ed016695db 100644 --- a/docs/src/content/docs/reference/scripts/retreival.md +++ b/docs/src/content/docs/reference/scripts/retreival.md @@ -1,17 +1,17 @@ --- -title: Retreival +title: Retrieval sidebar: order: 10 --- -GenAIScript provides various utilities to retreive content and augment the prompt. This technique is typically referred as **RAG** (Retreival-Augmentation-Generation) in the literature. GenAIScript uses [llamaindex-ts](https://ts.llamaindex.ai/api/classes/VectorIndexRetriever) which supports many vector database vendors. +GenAIScript provides various utilities to retreive content and augment the prompt. This technique is typically referred as **RAG** (Retrieval-Augmentation-Generation) in the literature. GenAIScript uses [llamaindex-ts](https://ts.llamaindex.ai/api/classes/VectorIndexRetriever) which supports many vector database vendors. ## Search The `retreive.search` performs a embeddings search to find the most similar documents to the prompt. The search is performed using the [llamaindex-ts](https://ts.llamaindex.ai/api/classes/VectorIndexRetriever) library. ```js -const { files, fragments } = await retreival.search("cat dog", env.files) +const { files, fragments } = await retrieval.search("cat dog", env.files) def("RAG", files) ``` @@ -19,7 +19,7 @@ The `files` variable contains a list of files, with concatenated fragments, that ### Indexing -By default, the retreival uses [OpenAI text-embedding-ada-002](https://ts.llamaindex.ai/modules/embeddings/) embeddings. The first search might be slow as the files get indexed for the first time. +By default, the retrieval uses [OpenAI text-embedding-ada-002](https://ts.llamaindex.ai/modules/embeddings/) embeddings. The first search might be slow as the files get indexed for the first time. You can index your project using the [CLI](/genaiscript/reference/cli). @@ -29,7 +29,7 @@ genaiscript retreive index "src/**" :::tip -You can simulate an indexing command in Visual Studio Code by right-clicking on a folder and selecting **Retreival** > **Index**. Once indexed, you can test search using **Retreival** > **Search**. +You can simulate an indexing command in Visual Studio Code by right-clicking on a folder and selecting **Retrieval** > **Index**. Once indexed, you can test search using **Retrieval** > **Search**. ::: @@ -39,10 +39,10 @@ You can control the chunk size, overlap and model used for index files. You can ## Web Search -The `retreival.webSearch` performs a web search using a search engine API. You will need to provide API keys for the search engine you want to use. +The `retrieval.webSearch` performs a web search using a search engine API. You will need to provide API keys for the search engine you want to use. ```js -const { webPages } = await retreival.webSearch("cat dog") +const { webPages } = await retrieval.webSearch("cat dog") def("RAG", webPages) ``` diff --git a/docs/src/content/docs/reference/scripts/web-search.md b/docs/src/content/docs/reference/scripts/web-search.md index 35da834ecb..00eedd0be8 100644 --- a/docs/src/content/docs/reference/scripts/web-search.md +++ b/docs/src/content/docs/reference/scripts/web-search.md @@ -6,7 +6,7 @@ sidebar: order: 15 --- -The `retreival.webSearch` executes a web search using the Bing Web Search API. +The `retrieval.webSearch` executes a web search using the Bing Web Search API. ## Web Pages @@ -15,7 +15,7 @@ as an array of files, similarly to `env.files`. The content contains the summary snippet returned by the search engine. ```js -const { webPages } = await retreival.webSearch("microsoft") +const { webPages } = await retrieval.webSearch("microsoft") def("PAGES", webPages) ``` @@ -31,7 +31,7 @@ BING_SEARCH_API_KEY="your-api-key" ## Function -Add the [system.web_search](https://github.com/microsoft/genaiscript/blob/main/packages/core/src/genaisrc/system.web_search.genai.js) system script to register a [function](/genaiscript/reference/scripts/functions) that uses `retreival.webSearch`. +Add the [system.web_search](https://github.com/microsoft/genaiscript/blob/main/packages/core/src/genaisrc/system.web_search.genai.js) system script to register a [function](/genaiscript/reference/scripts/functions) that uses `retrieval.webSearch`. ```js script({ diff --git a/package.json b/package.json index f1f83fb851..6c26eceb05 100644 --- a/package.json +++ b/package.json @@ -37,12 +37,12 @@ "test:front-matter": "node packages/cli/built/genaiscript.cjs run front-matter SUPPORT.md", "test:pdf": "node packages/cli/built/genaiscript.cjs parse pdf packages/sample/src/rag/loremipsum.pdf", "test:docx": "node packages/cli/built/genaiscript.cjs parse docx packages/sample/src/rag/Document.docx", - "test:index": "node packages/cli/built/genaiscript.cjs retreival index \"packages/sample/src/rag/*\"", - "test:index:summary": "node packages/cli/built/genaiscript.cjs retreival index \"packages/sample/src/rag/*\" --summary", - "test:search": "node packages/cli/built/genaiscript.cjs retreival search lorem \"packages/sample/src/rag/*\"", - "test:search:summary": "node packages/cli/built/genaiscript.cjs retreival search lorem \"packages/sample/src/rag/*\" --summary", + "test:index": "node packages/cli/built/genaiscript.cjs retrieval index \"packages/sample/src/rag/*\"", + "test:index:summary": "node packages/cli/built/genaiscript.cjs retrieval index \"packages/sample/src/rag/*\" --summary", + "test:search": "node packages/cli/built/genaiscript.cjs retrieval search lorem \"packages/sample/src/rag/*\"", + "test:search:summary": "node packages/cli/built/genaiscript.cjs retrieval search lorem \"packages/sample/src/rag/*\" --summary", "test:codequery": "node packages/cli/built/genaiscript.cjs code query packages/core/src/progress.ts \"(interface_declaration) @i\"", - "test:tokens": "node packages/cli/built/genaiscript.cjs retreival tokens packages/sample/src/rag/*", + "test:tokens": "node packages/cli/built/genaiscript.cjs retrieval tokens packages/sample/src/rag/*", "serve": "node packages/cli/built/genaiscript.cjs serve", "docs": "cd docs && ./node_modules/.bin/astro dev --host", "build:docs": "cd docs && yarn build && yarn build:asw", diff --git a/packages/cli/src/build.ts b/packages/cli/src/build.ts index 5b820b7b1d..c36966004b 100644 --- a/packages/cli/src/build.ts +++ b/packages/cli/src/build.ts @@ -1,4 +1,9 @@ -import { GENAI_EXT, host, parseProject } from "genaiscript-core" +import { + GENAI_JS_GLOB, + GPSPEC_GLOB, + host, + parseProject, +} from "genaiscript-core" export async function buildProject(options?: { toolFiles?: string[] @@ -9,8 +14,8 @@ export async function buildProject(options?: { const { toolFiles, specFiles, - toolsPath = "**/*" + GENAI_EXT, - specsPath = "**/*.gpspec.md", + toolsPath = GENAI_JS_GLOB, + specsPath = GPSPEC_GLOB, } = options || {} const gpspecFiles = specFiles?.length diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 21ef69b7b2..2b7d5be199 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -19,11 +19,11 @@ import { runScript } from "./run" import { buildProject } from "./build" import { batchScript } from "./batch" import { - retreivalClear, - retreivalIndex, - retreivalSearch, - retreivalTokens, -} from "./retreival" + retrievalClear, + retrievalIndex, + retrievalSearch, + retrievalTokens, +} from "./retrieval" import { helpAll } from "./help" import { jsonl2json, parseDOCX, parseFence, parsePDF } from "./parse" import { listScripts } from "./scripts" @@ -154,8 +154,8 @@ export async function cli() { .description("List all available scripts in workspace") .action(listScripts) - const retreival = program.command("retreival").description("RAG support") - retreival + const retrieval = program.command("retrieval").description("RAG support") + retrieval .command("index") .description("Index a set of documents") .argument("", "Files to index") @@ -169,8 +169,8 @@ export async function cli() { "-sls, --split-long-sentences", "split long sentences (default true)" ) - .action(retreivalIndex) - retreival + .action(retrievalIndex) + retrieval .command("search") .description("Search index") .arguments(" [files...]") @@ -178,13 +178,13 @@ export async function cli() { .option("-tk, --top-k ", "maximum number of embeddings") .option("-n, --name ", "index name") // .option("-s, --summary", "use LLM-generated summaries") - .action(retreivalSearch) - retreival + .action(retrievalSearch) + retrieval .command("clear") .description("Clear index to force re-indexing") .option("-n, --name ", "index name") // .option("-s, --summary", "use LLM-generated summaries") - .action(retreivalClear) + .action(retrievalClear) program .command("serve") @@ -219,7 +219,7 @@ export async function cli() { .description("Count tokens in a set of files") .arguments("") .option("-ef, --excluded-files ", "excluded files") - .action(retreivalTokens) + .action(retrievalTokens) parser .command("jsonl2json", "Converts JSONL files to a JSON file") .argument("", "input JSONL files") diff --git a/packages/cli/src/llamaindexretreival.ts b/packages/cli/src/llamaindexretrieval.ts similarity index 95% rename from packages/cli/src/llamaindexretreival.ts rename to packages/cli/src/llamaindexretrieval.ts index ca7434901a..f51e508304 100644 --- a/packages/cli/src/llamaindexretreival.ts +++ b/packages/cli/src/llamaindexretrieval.ts @@ -7,11 +7,11 @@ import { RETRIEVAL_DEFAULT_INDEX, RETRIEVAL_DEFAULT_MODEL, ResponseStatus, - RetreivalOptions, - RetreivalSearchOptions, - RetreivalSearchResponse, - RetreivalService, - RetreivalUpsertOptions, + RetrievalOptions, + RetrievalSearchOptions, + RetrievalSearchResponse, + RetrievalService, + RetrievalUpsertOptions, dotGenaiscriptPath, fileExists, installImport, @@ -64,7 +64,7 @@ async function tryImportLlamaIndex(trace: MarkdownTrace) { } } -export class LlamaIndexRetreivalService implements RetreivalService { +export class LlamaIndexRetrievalService implements RetrievalService { private module: PromiseType> private READERS: Record @@ -89,7 +89,7 @@ export class LlamaIndexRetreivalService implements RetreivalService { private getPersisDir(indexName: string, summary: boolean) { const persistDir = this.host.path.join( - dotGenaiscriptPath("retreival"), + dotGenaiscriptPath("retrieval"), summary ? "summary" : "full", indexName ) @@ -158,7 +158,7 @@ export class LlamaIndexRetreivalService implements RetreivalService { return serviceContext } - async clear(options?: RetreivalOptions) { + async clear(options?: RetrievalOptions) { const { indexName = RETRIEVAL_DEFAULT_INDEX, summary } = options || {} const persistDir = this.getPersisDir(indexName, summary) await this.host.deleteDirectory(persistDir) @@ -167,7 +167,7 @@ export class LlamaIndexRetreivalService implements RetreivalService { async upsert( filenameOrUrl: string, - options?: RetreivalUpsertOptions + options?: RetrievalUpsertOptions ): Promise { const { Document, VectorStoreIndex, SummaryIndex } = this.module const { content, mimeType, summary } = options ?? {} @@ -242,8 +242,8 @@ export class LlamaIndexRetreivalService implements RetreivalService { async search( text: string, - options?: RetreivalSearchOptions - ): Promise { + options?: RetrievalSearchOptions + ): Promise { const { topK = LLAMAINDEX_SIMILARITY_TOPK, minScore = LLAMAINDEX_MIN_SCORE, @@ -301,8 +301,8 @@ export class LlamaIndexRetreivalService implements RetreivalService { * @returns */ async embeddings( - options?: RetreivalOptions - ): Promise { + options?: RetrievalOptions + ): Promise { const { MetadataMode } = this.module const { storageContext } = await this.createStorageContext(options) const docs = await storageContext.docStore.docs() diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts index 4547bc00d6..b3bba5bfba 100644 --- a/packages/cli/src/nodehost.ts +++ b/packages/cli/src/nodehost.ts @@ -4,7 +4,7 @@ import { LogLevel, OAIToken, ReadFileOptions, - RetreivalService, + RetrievalService, ServerManager, ShellCallOptions, UTF8Decoder, @@ -21,7 +21,7 @@ import { glob } from "glob" import { debug, error, info, warn } from "./log" import { execa } from "execa" import { join } from "node:path" -import { LlamaIndexRetreivalService } from "./llamaindexretreival" +import { LlamaIndexRetrievalService } from "./llamaindexretrieval" import { createNodePath } from "./nodepath" class NodeServerManager implements ServerManager { @@ -36,13 +36,13 @@ class NodeServerManager implements ServerManager { export class NodeHost implements Host { userState: any = {} virtualFiles: Record = {} - retreival: RetreivalService + retrieval: RetrievalService readonly path = createNodePath() readonly server = new NodeServerManager() readonly fs = createFileSystem() constructor() { - this.retreival = new LlamaIndexRetreivalService(this) + this.retrieval = new LlamaIndexRetrievalService(this) } static install(dotEnvPath: string) { diff --git a/packages/cli/src/retreival.ts b/packages/cli/src/retrieval.ts similarity index 93% rename from packages/cli/src/retreival.ts rename to packages/cli/src/retrieval.ts index 1885713985..782928cfde 100644 --- a/packages/cli/src/retreival.ts +++ b/packages/cli/src/retrieval.ts @@ -12,7 +12,7 @@ import { } from "genaiscript-core" import { createProgressSpinner } from "./spinner" -export async function retreivalIndex( +export async function retrievalIndex( files: string[], options: { excludedFiles: string[] @@ -51,7 +51,7 @@ export async function retreivalIndex( }) } -export async function retreivalClear(options: { +export async function retrievalClear(options: { name: string summary: boolean }) { @@ -59,7 +59,7 @@ export async function retreivalClear(options: { await clearIndex({ indexName, summary }) } -export async function retreivalSearch( +export async function retrievalSearch( q: string, filesGlobs: string[], options: { @@ -84,7 +84,7 @@ export async function retreivalSearch( console.log(YAMLStringify(res)) } -export async function retreivalTokens( +export async function retrievalTokens( filesGlobs: string[], options: { excludedFiles: string[]; model: string } ) { diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index eb4bdb46b3..5e4dc45456 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -12,7 +12,7 @@ import { } from "genaiscript-core" export async function startServer(options: { port: string }) { - await host.retreival.init() + await host.retrieval.init() const port = parseInt(options.port) || SERVER_PORT const wss = new WebSocketServer({ port }) @@ -42,21 +42,21 @@ export async function startServer(options: { port: string }) { process.exit(0) break } - case "retreival.clear": - console.log(`retreival: clear`) - response = await host.retreival.clear(data.options) + case "retrieval.clear": + console.log(`retrieval: clear`) + response = await host.retrieval.clear(data.options) break - case "retreival.upsert": - console.log(`retreival: upsert ${data.filename}`) - response = await host.retreival.upsert( + case "retrieval.upsert": + console.log(`retrieval: upsert ${data.filename}`) + response = await host.retrieval.upsert( data.filename, data.options ) break - case "retreival.search": - console.log(`retreival: search ${data.text}`) + case "retrieval.search": + console.log(`retrieval: search ${data.text}`) console.debug(YAMLStringify(data.options)) - response = await host.retreival.search( + response = await host.retrieval.search( data.text, data.options ) diff --git a/packages/core/prompt_type.d.ts b/packages/core/prompt_type.d.ts index 43821b84ef..16bab0af0d 100644 --- a/packages/core/prompt_type.d.ts +++ b/packages/core/prompt_type.d.ts @@ -79,9 +79,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 5e43d5776f..fe81283d7d 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -7,11 +7,16 @@ export const TOOL_ID = "genaiscript" export const GENAISCRIPT_FOLDER = "." + TOOL_ID export const CLI_JS = TOOL_ID + ".cjs" export const GENAI_SRC = "genaisrc" -export const GENAI_EXT = ".genai.js" +export const GENAI_JS_EXT = ".genai.js" +export const GENAI_MJS_EXT = ".genai.mjs" +export const GENAI_JS_GLOB = "**/*.genai.*js" +export const GENAI_JS_REGEX = /\.genai\.m?js$/i +export const GPSPEC_REGEX = /\.gpspec\.md$/i +export const GPSPEC_GLOB = "**/*.gpspec.md" export const TOOL_NAME = "GenAIScript" export const SERVER_PORT = 8003 export const CLIENT_RECONNECT_DELAY = 2000 -export const RETREIVAL_PERSIST_DIR = "retreival" +export const RETREIVAL_PERSIST_DIR = "retrieval" export const HIGHLIGHT_LENGTH = 4000 export const DEFAULT_MODEL = "gpt-4" export const DEFAULT_TEMPERATURE = 0.2 // 0.0-2.0, defaults to 1.0 @@ -34,9 +39,6 @@ export const ICON_LOGO_NAME = "genaiscript-logo" export const SARIFF_RULEID_PREFIX = "genascript/" export const SARIFF_BUILDER_URL = "https://github.com/microsoft/genaiscript/" export const SARIFF_BUILDER_TOOL_DRIVER_NAME = TOOL_ID -export const GENAI_JS_REGEX = /\.genai\.js$/i -export const GPSPEC_REGEX = /\.gpspec\.md$/i - export const UNHANDLED_ERROR_CODE = -1 export const ANNOTATION_ERROR_CODE = -2 diff --git a/packages/core/src/copy.ts b/packages/core/src/copy.ts index 9509e812d1..cccc3e0995 100644 --- a/packages/core/src/copy.ts +++ b/packages/core/src/copy.ts @@ -1,12 +1,12 @@ import { PromptTemplate } from "./ast" -import { GENAI_EXT, GENAI_SRC } from "./constants" +import { GENAI_JS_EXT, GENAI_SRC } from "./constants" import { host } from "./host" import { fileExists, writeText } from "./fs" function promptPath(id: string) { const prompts = host.resolvePath(host.projectFolder(), GENAI_SRC) if (id === null) return prompts - return host.resolvePath(prompts, id + GENAI_EXT) + return host.resolvePath(prompts, id + GENAI_JS_EXT) } export async function copyPrompt( t: PromptTemplate, diff --git a/packages/core/src/expander.ts b/packages/core/src/expander.ts index d8d7fe419e..d68860995f 100644 --- a/packages/core/src/expander.ts +++ b/packages/core/src/expander.ts @@ -14,6 +14,7 @@ import { ChatCompletionSystemMessageParam, toChatCompletionUserMessage, } from "./chat" +import { importPrompt } from "./importprompt" const defaultTopP: number = undefined const defaultSeed: number = undefined @@ -91,13 +92,21 @@ async function callExpander( let outputProcessors: PromptOutputProcessorHandler[] = [] let aici: AICIRequest + const logCb = (msg: any) => { + logs += msg + "\n" + } + try { - await evalPrompt(ctx, r, { - sourceMaps: true, - logCb: (msg: any) => { - logs += msg + "\n" - }, - }) + if (/^export\s+default\s+/m.test(r.jsSource)) { + if (!/\.mjs$/i.test(r.filename)) + throw new Error("export default requires .mjs file") + await importPrompt(ctx, r, { logCb }) + } else { + await evalPrompt(ctx, r, { + sourceMaps: true, + logCb, + }) + } const node = ctx.node if (!r.aici) { const { diff --git a/packages/core/src/genaisrc/jsconfig.json b/packages/core/src/genaisrc/jsconfig.json index 92c74049fa..077ce1fcb8 100644 --- a/packages/core/src/genaisrc/jsconfig.json +++ b/packages/core/src/genaisrc/jsconfig.json @@ -1,6 +1,4 @@ { - // for now, drop this file in prompt folder - // in future, we'll add a TS Server plugin that will fake this file presence "compilerOptions": { "lib": ["ES2022"], "checkJs": true, @@ -9,9 +7,5 @@ "moduleDetection": "force", "allowJs": true }, - "include": [ - "*.js", - "*.ts", - "../../prompt_type.d.ts" - ] -} \ No newline at end of file + "include": ["*.js", "../../prompt_type.d.ts"] +} diff --git a/packages/core/src/genaisrc/system.web_search.genai.js b/packages/core/src/genaisrc/system.web_search.genai.js index 215192cfb7..b398c463ad 100644 --- a/packages/core/src/genaisrc/system.web_search.genai.js +++ b/packages/core/src/genaisrc/system.web_search.genai.js @@ -19,7 +19,7 @@ defFunction( }, async (args) => { const { q } = args - const { webPages } = await retreival.webSearch(q) + const { webPages } = await retrieval.webSearch(q) return YAML.stringify( webPages.map((f) => ({ url: f.filename, diff --git a/packages/core/src/host.ts b/packages/core/src/host.ts index ea8057a507..ae9adb1217 100644 --- a/packages/core/src/host.ts +++ b/packages/core/src/host.ts @@ -58,7 +58,7 @@ export interface ShellCallOptions { exitcodefile: string } -export interface RetreivalClientOptions { +export interface RetrievalClientOptions { progress?: Progress token?: CancellationToken trace?: MarkdownTrace @@ -70,18 +70,18 @@ export interface ResponseStatus { status?: number } -export interface RetreivalOptions { +export interface RetrievalOptions { indexName?: string summary?: boolean } -export interface RetreivalSearchOptions extends RetreivalOptions { +export interface RetrievalSearchOptions extends RetrievalOptions { files?: string[] topK?: number minScore?: number } -export interface RetreivalUpsertOptions extends RetreivalOptions { +export interface RetrievalUpsertOptions extends RetrievalOptions { content?: string mimeType?: string model?: string @@ -90,7 +90,7 @@ export interface RetreivalUpsertOptions extends RetreivalOptions { splitLongSentences?: boolean } -export type RetreivalSearchResponse = ResponseStatus & { +export type RetrievalSearchResponse = ResponseStatus & { results: { filename: string id: string @@ -99,17 +99,17 @@ export type RetreivalSearchResponse = ResponseStatus & { }[] } -export interface RetreivalService { +export interface RetrievalService { init(trace?: MarkdownTrace): Promise - clear(options?: RetreivalOptions): Promise + clear(options?: RetrievalOptions): Promise upsert( filenameOrUrl: string, - options?: RetreivalUpsertOptions + options?: RetrievalUpsertOptions ): Promise search( text: string, - options?: RetreivalSearchOptions - ): Promise + options?: RetrievalSearchOptions + ): Promise } export type HighlightResponse = ResponseStatus & { response: string } @@ -124,7 +124,7 @@ export interface ServerManager { export interface Host { userState: any - retreival: RetreivalService + retrieval: RetrievalService server: ServerManager path: Path fs: FileSystem diff --git a/packages/core/src/importprompt.ts b/packages/core/src/importprompt.ts new file mode 100644 index 0000000000..b701a545eb --- /dev/null +++ b/packages/core/src/importprompt.ts @@ -0,0 +1,43 @@ +import { assert } from "console" +import { host } from "./host" +import { logError } from "./util" + +function resolveGlobal(): any { + if (typeof window !== "undefined") + return window // Browser environment + else if (typeof self !== "undefined") return self + else if (typeof global !== "undefined") return global // Node.js environment + throw new Error("Could not find global") +} + +export async function importPrompt( + ctx0: PromptContext, + r: PromptTemplate, + options?: { + logCb?: (msg: string) => void + } +) { + const { filename } = r + if (!filename) throw new Error("filename is required") + + const glb: any = resolveGlobal() + try { + for (const field of Object.keys(ctx0)) { + assert(!glb[field]) + glb[field] = (ctx0 as any)[field] + } + + const modulePath = filename.startsWith("/") ? filename : host.path.join(host.projectFolder(), filename) + const module = await import(modulePath) + const main = module.default + if (!main) throw new Error("default import function missing") + if (typeof main !== "function") + throw new Error("default export must be a function") + await main(ctx0) + } catch (err) { + logError(err) + throw err + } finally { + for (const field of Object.keys(ctx0)) delete glb[field] + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 09e381633e..36f3cbf416 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -25,7 +25,7 @@ export * from "./yaml" export * from "./schema" export * from "./version" export * from "./toml" -export * from "./retreival" +export * from "./retrieval" export * from "./progress" export * from "./server/client" export * from "./server/messages" @@ -44,6 +44,7 @@ export * from "./frontmatter" export * from "./promptcontext" export * from "./runpromptcontext" export * from "./evalprompt" +export * from "./importprompt" export * from "./promptrunner" export * from "./clihelp" export * from "./openai" diff --git a/packages/core/src/prompt_template.d.ts b/packages/core/src/prompt_template.d.ts index 5b6c1856d2..68b2e70828 100644 --- a/packages/core/src/prompt_template.d.ts +++ b/packages/core/src/prompt_template.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index 42fe103764..5ec3d9ef1a 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -5,7 +5,7 @@ import { host } from "./host" import { MarkdownTrace } from "./trace" import { YAMLParse, YAMLStringify } from "./yaml" import { createParsers } from "./parsers" -import { upsert, search } from "./retreival" +import { upsert, search } from "./retrieval" import { readText } from "./fs" import { PromptNode, @@ -74,10 +74,10 @@ export function createPromptContext( const path = host.path const fs = host.fs - const retreival: Retreival = { + const retrieval: Retrieval = { webSearch: async (q) => { try { - trace.startDetails(`🌐 retreival web search \`${q}\``) + trace.startDetails(`🌐 retrieval web search \`${q}\``) const { webPages } = (await bingSearch(q, { trace })) || {} return { webPages: webPages?.value?.map( @@ -99,7 +99,7 @@ export function createPromptContext( search: async (q, files, searchOptions) => { searchOptions = searchOptions || {} try { - trace.startDetails(`🔍 retreival search \`${q}\``) + trace.startDetails(`🔍 retrieval search \`${q}\``) if (!files?.length) { trace.error("no files provided") return { files: [], fragments: [] } @@ -190,7 +190,7 @@ export function createPromptContext( CSV, INI, AICI, - retreival, + retrieval, defImages, defSchema, defOutput, diff --git a/packages/core/src/retreival.ts b/packages/core/src/retrieval.ts similarity index 74% rename from packages/core/src/retreival.ts rename to packages/core/src/retrieval.ts index c7ee09dc4d..28c8d34101 100644 --- a/packages/core/src/retreival.ts +++ b/packages/core/src/retrieval.ts @@ -1,8 +1,8 @@ import { - RetreivalClientOptions, - RetreivalOptions, - RetreivalSearchOptions, - RetreivalUpsertOptions, + RetrievalClientOptions, + RetrievalOptions, + RetrievalSearchOptions, + RetrievalUpsertOptions, host, } from "./host" import { lookupMime } from "./mime" @@ -18,21 +18,21 @@ export function isIndexable(filename: string) { } export async function clearIndex( - options?: RetreivalClientOptions & RetreivalOptions + options?: RetrievalClientOptions & RetrievalOptions ): Promise { const { trace } = options || {} - await host.retreival.init(trace) - await host.retreival.clear(options) + await host.retrieval.init(trace) + await host.retrieval.clear(options) } export async function upsert( fileOrUrls: (string | LinkedFile)[], - options?: RetreivalClientOptions & RetreivalUpsertOptions + options?: RetrievalClientOptions & RetrievalUpsertOptions ) { if (!fileOrUrls?.length) return const { progress, trace, token, ...rest } = options || {} - const retreival = host.retreival - await retreival.init(trace) + const retrieval = host.retrieval + await retrieval.init(trace) const files: LinkedFile[] = fileOrUrls.map((f) => typeof f === "string" ? { filename: f } : f ) @@ -43,7 +43,7 @@ export async function upsert( count: ++count, message: f.filename, }) - const { ok } = await retreival.upsert(f.filename, { + const { ok } = await retrieval.upsert(f.filename, { content: f.content, ...rest, }) @@ -55,22 +55,22 @@ export async function upsert( } } -export interface RetreivalSearchResult { +export interface RetrievalSearchResult { files: LinkedFile[] fragments: LinkedFile[] } export async function search( q: string, - options?: RetreivalClientOptions & RetreivalSearchOptions -): Promise { + options?: RetrievalClientOptions & RetrievalSearchOptions +): Promise { const { trace, token, ...rest } = options || {} const files: LinkedFile[] = [] - const retreival = host.retreival - await host.retreival.init(trace) + const retrieval = host.retrieval + await host.retrieval.init(trace) if (token?.isCancellationRequested) return { files, fragments: [] } - const { results } = await retreival.search(q, rest) + const { results } = await retrieval.search(q, rest) const fragments = (results || []).map((r) => { const { id, filename, text } = r diff --git a/packages/core/src/server/client.ts b/packages/core/src/server/client.ts index cceb123d7c..37de996304 100644 --- a/packages/core/src/server/client.ts +++ b/packages/core/src/server/client.ts @@ -1,24 +1,24 @@ import { CLIENT_RECONNECT_DELAY } from "../constants" import { ResponseStatus, - RetreivalOptions, - RetreivalSearchOptions, - RetreivalSearchResponse, - RetreivalService, - RetreivalUpsertOptions, + RetrievalOptions, + RetrievalSearchOptions, + RetrievalSearchResponse, + RetrievalService, + RetrievalUpsertOptions, host, } from "../host" import { assert } from "../util" import { RequestMessage, RequestMessages, - RetreivalClear, - RetreivalSearch, - RetreivalUpsert, + RetrievalClear, + RetrievalSearch, + RetrievalUpsert, ServerVersion, } from "./messages" -export class WebSocketClient implements RetreivalService { +export class WebSocketClient implements RetrievalService { private awaiters: Record< string, { resolve: (data: any) => void; reject: (error: unknown) => void } @@ -106,9 +106,9 @@ export class WebSocketClient implements RetreivalService { return res.version } - async clear(options: RetreivalOptions): Promise { - const res = await this.queue({ - type: "retreival.clear", + async clear(options: RetrievalOptions): Promise { + const res = await this.queue({ + type: "retrieval.clear", options, }) return res.response @@ -116,18 +116,18 @@ export class WebSocketClient implements RetreivalService { async search( text: string, - options?: RetreivalSearchOptions - ): Promise { - const res = await this.queue({ - type: "retreival.search", + options?: RetrievalSearchOptions + ): Promise { + const res = await this.queue({ + type: "retrieval.search", text, options, }) return res.response } - async upsert(filename: string, options?: RetreivalUpsertOptions) { - const res = await this.queue({ - type: "retreival.upsert", + async upsert(filename: string, options?: RetrievalUpsertOptions) { + const res = await this.queue({ + type: "retrieval.upsert", filename, options, }) diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts index 3bc3d8c94d..2dcfeffdd8 100644 --- a/packages/core/src/server/messages.ts +++ b/packages/core/src/server/messages.ts @@ -1,10 +1,10 @@ import { HighlightResponse, ResponseStatus, - RetreivalOptions, - RetreivalSearchOptions, - RetreivalSearchResponse, - RetreivalUpsertOptions, + RetrievalOptions, + RetrievalSearchOptions, + RetrievalSearchResponse, + RetrievalUpsertOptions, } from "../host" export interface RequestMessage { @@ -17,9 +17,9 @@ export interface ServerKill extends RequestMessage { type: "server.kill" } -export interface RetreivalClear extends RequestMessage { - type: "retreival.clear" - options?: RetreivalOptions +export interface RetrievalClear extends RequestMessage { + type: "retrieval.clear" + options?: RetrievalOptions } export interface ServerVersion extends RequestMessage { @@ -27,22 +27,22 @@ export interface ServerVersion extends RequestMessage { version?: string } -export interface RetreivalUpsert extends RequestMessage { - type: "retreival.upsert" +export interface RetrievalUpsert extends RequestMessage { + type: "retrieval.upsert" filename: string - options?: RetreivalUpsertOptions + options?: RetrievalUpsertOptions } -export interface RetreivalSearch extends RequestMessage { - type: "retreival.search" +export interface RetrievalSearch extends RequestMessage { + type: "retrieval.search" text: string - options?: RetreivalSearchOptions - response?: RetreivalSearchResponse + options?: RetrievalSearchOptions + response?: RetrievalSearchResponse } export type RequestMessages = | ServerKill - | RetreivalClear - | RetreivalUpsert - | RetreivalSearch + | RetrievalClear + | RetrievalUpsert + | RetrievalSearch | ServerVersion diff --git a/packages/core/src/template.ts b/packages/core/src/template.ts index 6a6c662487..3df5e587eb 100644 --- a/packages/core/src/template.ts +++ b/packages/core/src/template.ts @@ -1,7 +1,5 @@ import { Project, Fragment, PromptTemplate } from "./ast" -import { randomRange, sha256string } from "./util" import { BUILTIN_PREFIX } from "./constants" -import { evalPrompt } from "./evalprompt" import { JSON5TryParse } from "./json5" function templateIdFromFileName(filename: string) { return filename diff --git a/packages/sample/genaisrc/bingsearch.genai.js b/packages/sample/genaisrc/bingsearch.genai.js index 4550057890..1f015a7406 100644 --- a/packages/sample/genaisrc/bingsearch.genai.js +++ b/packages/sample/genaisrc/bingsearch.genai.js @@ -2,5 +2,5 @@ script({ title: "bing search", }) -const { webPages } = await retreival.webSearch("microsoft") +const { webPages } = await retrieval.webSearch("microsoft") def("RES", webPages, { language: "json" }) diff --git a/packages/sample/genaisrc/code-outline.genai.js b/packages/sample/genaisrc/code-outline.genai.js deleted file mode 100644 index e524a40b73..0000000000 --- a/packages/sample/genaisrc/code-outline.genai.js +++ /dev/null @@ -1,8 +0,0 @@ -script({ - title: "code outline", -}) - -const outline = await retreival.outline(env.files) -const code = def("CODE", outline) - -$`Summarize the code in ${code}.` diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/genaisrc/rag.genai.js b/packages/sample/genaisrc/rag.genai.js index 469dae380d..049dd81611 100644 --- a/packages/sample/genaisrc/rag.genai.js +++ b/packages/sample/genaisrc/rag.genai.js @@ -7,7 +7,7 @@ $`You are a helpful assistant. Summarize the files.` def( "MARKDOWN", ( - await retreival.search( + await retrieval.search( "markdown", env.files.filter((f) => f.filename.endsWith(".md")) ) @@ -16,7 +16,7 @@ def( def( "PDF", ( - await retreival.search( + await retrieval.search( "lorem ipsum", env.files.filter((f) => f.filename.endsWith(".pdf")) ) @@ -25,10 +25,10 @@ def( def( "WORD", ( - await retreival.search( + await retrieval.search( "lorem ipsum", env.files.filter((f) => f.filename.endsWith(".docx")) ) ).files ) -def("ALL", (await retreival.search("lorem ipsum", env.files)).files) +def("ALL", (await retrieval.search("lorem ipsum", env.files)).files) diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/genaisrc/summarize-import.genai.mjs b/packages/sample/genaisrc/summarize-import.genai.mjs new file mode 100644 index 0000000000..1fc6217f65 --- /dev/null +++ b/packages/sample/genaisrc/summarize-import.genai.mjs @@ -0,0 +1,12 @@ +import { parse } from "ini" + +script({ + title: "summarize all files using import" +}) + +export default async function () { + const res = parse("x = 1\ny = 2") + console.log(res) + def("FILE", env.files) + $`Summarize each file. Be concise.` +} \ No newline at end of file diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 56221b87c5..e40985d6aa 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -788,7 +788,7 @@ interface SearchResult { webPages: LinkedFile[] } -interface Retreival { +interface Retrieval { /** * Executers a Bing web search. Requires to configure the BING_SEARCH_API_KEY secret. * @param query @@ -1027,7 +1027,7 @@ interface PromptContext extends RunPromptContext { env: ExpansionVariables path: Path parsers: Parsers - retreival: Retreival + retrieval: Retrieval fs: FileSystem YAML: YAML CSV: CSV @@ -1116,9 +1116,9 @@ declare var path: Path declare var parsers: Parsers /** - * Retreival Augmented Generation services + * Retrieval Augmented Generation services */ -declare var retreival: Retreival +declare var retrieval: Retrieval /** * Access to file system operation on the current workspace. diff --git a/packages/vscode/README.md b/packages/vscode/README.md index c261a4bb98..f48fedeee2 100644 --- a/packages/vscode/README.md +++ b/packages/vscode/README.md @@ -45,7 +45,7 @@ const { pages } = await parsers.PDF(env.files[0]) ```js wrap // embedding vector index and search -const { files } = await retreival.search("cats", env.files) +const { files } = await retrieval.search("cats", env.files) ``` - 🚀 Automate using the [CLI](https://microsoft.github.io/genaiscript/reference/cli). diff --git a/packages/vscode/package.json b/packages/vscode/package.json index 50753e9690..bb042d2ad1 100644 --- a/packages/vscode/package.json +++ b/packages/vscode/package.json @@ -182,26 +182,26 @@ "group": "genaiscript@1" }, { - "submenu": "genaiscript.explorer.retreival", + "submenu": "genaiscript.explorer.retrieval", "when": "explorerViewletVisible && !inDebugMode && !(resourceFilename =~ /\\.genai\\.js$/)", "group": "genaiscript@2" } ], - "genaiscript.explorer.retreival": [ + "genaiscript.explorer.retrieval": [ { - "command": "genaiscript.retreival.index", + "command": "genaiscript.retrieval.index", "when": "explorerViewletVisible" }, { - "command": "genaiscript.retreival.search", + "command": "genaiscript.retrieval.search", "when": "explorerViewletVisible" } ] }, "submenus": [ { - "id": "genaiscript.explorer.retreival", - "label": "Retreival" + "id": "genaiscript.explorer.retrieval", + "label": "Retrieval" } ], "configuration": [ @@ -308,18 +308,18 @@ "category": "GenAIScript" }, { - "command": "genaiscript.retreival.search", + "command": "genaiscript.retrieval.search", "title": "Search", "category": "GenAIScript" }, { - "command": "genaiscript.retreival.index", + "command": "genaiscript.retrieval.index", "title": "Index", "category": "GenAIScript" }, { - "command": "genaiscript.retreival.clear", - "title": "Clear retreival index", + "command": "genaiscript.retrieval.clear", + "title": "Clear retrieval index", "category": "GenAIScript" } ] diff --git a/packages/vscode/src/extension.ts b/packages/vscode/src/extension.ts index 308439a6ca..2f26097ce1 100644 --- a/packages/vscode/src/extension.ts +++ b/packages/vscode/src/extension.ts @@ -10,14 +10,14 @@ import { activatePromptCommands, commandButtons } from "./promptcommands" import { activateOpenAIRequestTreeDataProvider } from "./openairequesttree" import { activateAIRequestTreeDataProvider } from "./airequesttree" //import { activateChatParticipant } from "./chat/participant" -import { activateRetreivalCommands } from "./retreivalcommands" +import { activateRetrievalCommands } from "./retrievalcommands" // import { activateTokensStatusBar } from "./tokenstatusbar" export async function activate(context: ExtensionContext) { const state = new ExtensionState(context) activatePromptCommands(state) activateFragmentCommands(state) - activateRetreivalCommands(state) + activateRetrievalCommands(state) activateMarkdownTextDocumentContentProvider(state) activatePrompTreeDataProvider(state) activateAIRequestTreeDataProvider(state) diff --git a/packages/vscode/src/markdowndocumentprovider.ts b/packages/vscode/src/markdowndocumentprovider.ts index bca0a95537..da479f5357 100644 --- a/packages/vscode/src/markdowndocumentprovider.ts +++ b/packages/vscode/src/markdowndocumentprovider.ts @@ -8,7 +8,7 @@ import { } from "./state" import { showMarkdownPreview } from "./markdown" import { - GENAI_EXT, + GENAI_JS_EXT, YAMLStringify, BUILTIN_PREFIX, CACHE_AIREQUEST_PREFIX, @@ -166,7 +166,7 @@ export function infoUri(path: string) { export function builtinPromptUri(id: string) { return vscode.Uri.from({ scheme: SCHEME, - path: BUILTIN_PREFIX + id + GENAI_EXT, + path: BUILTIN_PREFIX + id + GENAI_JS_EXT, }) } diff --git a/packages/vscode/src/retreivalcommands.ts b/packages/vscode/src/retrievalcommands.ts similarity index 90% rename from packages/vscode/src/retreivalcommands.ts rename to packages/vscode/src/retrievalcommands.ts index 6edaa38d12..9d1d13b2db 100644 --- a/packages/vscode/src/retreivalcommands.ts +++ b/packages/vscode/src/retrievalcommands.ts @@ -6,14 +6,14 @@ import { TOOL_NAME, clearIndex, isIndexable, - search as retreivalSearch, + search as retrievalSearch, ICON_LOGO_NAME, RETRIEVAL_DEFAULT_INDEX, } from "genaiscript-core" import { infoUri } from "./markdowndocumentprovider" import { showMarkdownPreview } from "./markdown" -export function activateRetreivalCommands(state: ExtensionState) { +export function activateRetrievalCommands(state: ExtensionState) { const { context } = state const { subscriptions } = context @@ -77,7 +77,7 @@ export function activateRetreivalCommands(state: ExtensionState) { async (progress, token) => { try { state.lastSearch = undefined - const res = await retreivalSearch(keywords, { + const res = await retrievalSearch(keywords, { files, token, }) @@ -102,16 +102,16 @@ export function activateRetreivalCommands(state: ExtensionState) { iconPath: new vscode.ThemeIcon(ICON_LOGO_NAME), }) terminal.sendText( - `node "${state.cliJsPath}" retreival index "${host.path.join(vscode.workspace.asRelativePath(uri.fsPath), "**")}" --name ${indexName}` + `node "${state.cliJsPath}" retrieval index "${host.path.join(vscode.workspace.asRelativePath(uri.fsPath), "**")}" --name ${indexName}` ) terminal.show() } subscriptions.push( - vscode.commands.registerCommand("genaiscript.retreival.index", index), - vscode.commands.registerCommand("genaiscript.retreival.search", search), + vscode.commands.registerCommand("genaiscript.retrieval.index", index), + vscode.commands.registerCommand("genaiscript.retrieval.search", search), vscode.commands.registerCommand( - "genaiscript.retreival.clear", + "genaiscript.retrieval.clear", clearIndex ) ) diff --git a/packages/vscode/src/servermanager.ts b/packages/vscode/src/servermanager.ts index 593e81734a..ab77e974ad 100644 --- a/packages/vscode/src/servermanager.ts +++ b/packages/vscode/src/servermanager.ts @@ -1,6 +1,6 @@ import { ICON_LOGO_NAME, - RetreivalService, + RetrievalService, SERVER_PORT, ServerManager, TOOL_NAME, @@ -59,7 +59,7 @@ export class TerminalServerManager implements ServerManager { this._terminal.show() } - get retreival(): RetreivalService { + get retrieval(): RetrievalService { return this.client } diff --git a/packages/vscode/src/state.ts b/packages/vscode/src/state.ts index 7735b38f52..db3c9d0fd9 100644 --- a/packages/vscode/src/state.ts +++ b/packages/vscode/src/state.ts @@ -26,11 +26,11 @@ import { dotGenaiscriptPath, CLI_JS, TOOL_ID, - GENAI_EXT, TOOL_NAME, - RetreivalSearchResult, + RetrievalSearchResult, AbortSignalCancellationToken, GENAI_JS_REGEX, + GENAI_JS_GLOB, } from "genaiscript-core" import { ExtensionContext } from "vscode" import { VSCodeHost } from "./vshost" @@ -147,7 +147,7 @@ export class ExtensionState extends EventTarget { undefined readonly output: vscode.LogOutputChannel - lastSearch: RetreivalSearchResult + lastSearch: RetrievalSearchResult constructor(public readonly context: ExtensionContext) { super() @@ -451,7 +451,7 @@ ${e.message}` } async fixPromptDefinitions() { - const prompts = await vscode.workspace.findFiles("**/*" + GENAI_EXT) + const prompts = await vscode.workspace.findFiles(GENAI_JS_GLOB) const folders = new Set(prompts.map((f) => Utils.dirname(f).fsPath)) for (const folder of folders) { const f = vscode.Uri.file(folder) @@ -475,7 +475,7 @@ ${e.message}` } async findScripts() { - const scriptFiles = await findFiles("**/*" + GENAI_EXT) + const scriptFiles = await findFiles(GENAI_JS_GLOB) return scriptFiles } diff --git a/packages/vscode/src/vshost.ts b/packages/vscode/src/vshost.ts index 77d0cdfa5d..259619eb1f 100644 --- a/packages/vscode/src/vshost.ts +++ b/packages/vscode/src/vshost.ts @@ -36,8 +36,8 @@ export class VSCodeHost extends EventTarget implements Host { this.state.context.subscriptions.push(this) } - get retreival() { - return this.server.retreival + get retrieval() { + return this.server.retrieval } get context() {