From 5940c30a1aa7751b7f8df175131f6a224c42c1a7 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Fri, 16 Aug 2024 18:55:00 +0000 Subject: [PATCH 1/5] moving tests to ollama --- .vscode/settings.json | 1 + packages/core/src/types/prompt_template.d.ts | 9 ++++++++- packages/sample/genaisrc/rag.genai.js | 10 ++++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9c0d09742d..acd266652a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,7 @@ "markdownify", "memorystream", "millis", + "nomic", "ollama", "openai", "promptdom", diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 053875a551..c186a10c90 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/genaisrc/rag.genai.js b/packages/sample/genaisrc/rag.genai.js index 1bfec958a4..46e12bf9ea 100644 --- a/packages/sample/genaisrc/rag.genai.js +++ b/packages/sample/genaisrc/rag.genai.js @@ -10,25 +10,27 @@ script({ $`You are a helpful assistant. Summarize the files in MARKDOWN, PDF, WORD and ALL.` +const embeddingsModel = env.vars.embeddingsModel || "ollama:nomic-embed-text" + def( "MARKDOWN", await retrieval.vectorSearch( "markdown", - env.files.filter((f) => f.filename.endsWith(".md")) + env.files.filter((f) => f.filename.endsWith(".md")), { embeddingsModel } ) ) def( "PDF", await retrieval.vectorSearch( "lorem", - env.files.filter((f) => f.filename.endsWith(".pdf")) + env.files.filter((f) => f.filename.endsWith(".pdf")), { embeddingsModel } ) ) def( "WORD", await retrieval.vectorSearch( "word", - env.files.filter((f) => f.filename.endsWith(".docx")) + env.files.filter((f) => f.filename.endsWith(".docx")), { embeddingsModel } ) ) -def("ALL", await retrieval.vectorSearch("lorem", env.files)) +def("ALL", await retrieval.vectorSearch("lorem", env.files), { embeddingsModel }) From 564e6ded07b15847f3328aeaac1ac9e3e2b08d08 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Fri, 16 Aug 2024 11:55:49 -0700 Subject: [PATCH 2/5] erwfresh gen --- docs/genaisrc/genaiscript.d.ts | 9 ++++++++- genaisrc/genaiscript.d.ts | 9 ++++++++- packages/core/src/genaisrc/genaiscript.d.ts | 9 ++++++++- packages/sample/genaisrc/genaiscript.d.ts | 9 ++++++++- packages/sample/genaisrc/node/genaiscript.d.ts | 9 ++++++++- packages/sample/genaisrc/python/genaiscript.d.ts | 9 ++++++++- packages/sample/genaisrc/style/genaiscript.d.ts | 9 ++++++++- packages/sample/src/aici/genaiscript.d.ts | 9 ++++++++- packages/sample/src/errors/genaiscript.d.ts | 9 ++++++++- packages/sample/src/makecode/genaiscript.d.ts | 9 ++++++++- packages/sample/src/tla/genaiscript.d.ts | 9 ++++++++- packages/sample/src/vision/genaiscript.d.ts | 9 ++++++++- slides/genaisrc/genaiscript.d.ts | 9 ++++++++- 13 files changed, 104 insertions(+), 13 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 26c996982c..fc2705b809 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -187,7 +187,14 @@ interface EmbeddingsModelConnectionOptions { /** * LLM model to use for embeddings. */ - embeddingsModel?: "openai:text-embedding-ada-002" | string + embeddingsModel?: OptionsOrString< + "openai:text-embedding-3-small", + "openai:text-embedding-3-large", + "openai:text-embedding-ada-002", + "github:text-embedding-3-small", + "github:text-embedding-3-large", + "ollama:nomic-embed-text" + > } interface EmbeddingsModelOptions extends EmbeddingsModelConnectionOptions {} From d6a5d6f0726cea9eba072797cbe692f778d5e48a Mon Sep 17 00:00:00 2001 From: pelikhan Date: Fri, 16 Aug 2024 12:15:51 -0700 Subject: [PATCH 3/5] configurable vector search --- .../content/docs/reference/scripts/system.mdx | 4 +++- .../system.retrieval_vector_search.genai.js | 4 +++- packages/core/src/models.ts | 2 +- packages/core/src/vectorsearch.ts | 5 +++-- packages/sample/genaisrc/rag.genai.js | 16 ++++++++++++---- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index 1633639664..c1fff729c1 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -710,6 +710,8 @@ system({ "Function to do a search using embeddings vector similarity distance.", }) +const embeddingsModel = env.var.embeddingsModel || undefined + defTool( "retrieval_vector_search", "Search files using embeddings and similarity distance.", @@ -737,7 +739,7 @@ defTool( const res = await retrieval.vectorSearch( q, files.map((filename) => ({ filename })) - ) + , { embeddingsModel }) return YAML.stringify(res.map(({ filename }) => filename)) } ) diff --git a/packages/core/src/genaisrc/system.retrieval_vector_search.genai.js b/packages/core/src/genaisrc/system.retrieval_vector_search.genai.js index 51be2c02f2..6efade50b9 100644 --- a/packages/core/src/genaisrc/system.retrieval_vector_search.genai.js +++ b/packages/core/src/genaisrc/system.retrieval_vector_search.genai.js @@ -4,6 +4,8 @@ system({ "Function to do a search using embeddings vector similarity distance.", }) +const embeddingsModel = env.var.embeddingsModel || undefined + defTool( "retrieval_vector_search", "Search files using embeddings and similarity distance.", @@ -31,7 +33,7 @@ defTool( const res = await retrieval.vectorSearch( q, files.map((filename) => ({ filename })) - ) + , { embeddingsModel }) return YAML.stringify(res.map(({ filename }) => filename)) } ) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index 8f60a058fa..bf706de187 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -101,7 +101,7 @@ export async function resolveModelConnectionInfo( const configuration = await host.getLanguageModelConfiguration( model, { - token: askToken, + token: withToken, signal, trace, } diff --git a/packages/core/src/vectorsearch.ts b/packages/core/src/vectorsearch.ts index 1d0e9f5d01..6486367da5 100644 --- a/packages/core/src/vectorsearch.ts +++ b/packages/core/src/vectorsearch.ts @@ -8,7 +8,7 @@ import { JSONLineCache } from "./cache" import { EmbeddingCreateParams, EmbeddingCreateResponse } from "./chattypes" import { LanguageModelConfiguration } from "./host" import { getConfigHeaders } from "./openai" -import { trimTrailingSlash } from "./util" +import { logVerbose, trimTrailingSlash } from "./util" import { TraceOptions } from "./trace" export interface EmbeddingsCacheKey { @@ -72,10 +72,11 @@ class OpenAIEmbeddings implements EmbeddingsModel { url = `${trimTrailingSlash(base)}/${model.replace(/\./g, "")}/embeddings?api-version=${AZURE_OPENAI_API_VERSION}` delete body.model } else { - url = `${base}/v1/embeddings` + url = `${base}/embeddings` } const fetch = await createFetch({ retryOn: [429] }) if (trace) traceFetchPost(trace, url, headers, body) + logVerbose(`embedding ${model}`) const resp = await fetch(url, { method: "POST", headers, diff --git a/packages/sample/genaisrc/rag.genai.js b/packages/sample/genaisrc/rag.genai.js index 46e12bf9ea..5a6fa05104 100644 --- a/packages/sample/genaisrc/rag.genai.js +++ b/packages/sample/genaisrc/rag.genai.js @@ -16,21 +16,29 @@ def( "MARKDOWN", await retrieval.vectorSearch( "markdown", - env.files.filter((f) => f.filename.endsWith(".md")), { embeddingsModel } + env.files.filter((f) => f.filename.endsWith(".md")), + { embeddingsModel } ) ) def( "PDF", await retrieval.vectorSearch( "lorem", - env.files.filter((f) => f.filename.endsWith(".pdf")), { embeddingsModel } + env.files.filter((f) => f.filename.endsWith(".pdf")), + { embeddingsModel } ) ) def( "WORD", await retrieval.vectorSearch( "word", - env.files.filter((f) => f.filename.endsWith(".docx")), { embeddingsModel } + env.files.filter((f) => f.filename.endsWith(".docx")), + { embeddingsModel } ) ) -def("ALL", await retrieval.vectorSearch("lorem", env.files), { embeddingsModel }) +def( + "ALL", + await retrieval.vectorSearch("lorem", env.files, { + embeddingsModel, + }) +) From 8e66494ed166d0fceea9f6b1ff7abcc6d71f88a6 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Fri, 16 Aug 2024 12:21:16 -0700 Subject: [PATCH 4/5] run rag in test suite --- .github/workflows/ollama.yml | 6 ++++-- packages/sample/genaisrc/rag.genai.js | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml index b5170be462..221d500f60 100644 --- a/.github/workflows/ollama.yml +++ b/.github/workflows/ollama.yml @@ -40,6 +40,8 @@ jobs: - name: download ollama docker run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama - name: run summarize-ollama-phi3 - run: yarn test:summarize --model ollama:phi3 --out ./temp + run: yarn test:summarize --model ollama:phi3 --out ./temp/summarize-ollama-phi3 + - name: run rag + run: yarn run:script rag --out ./temp/rag - name: test summarize-links-phi3 - run: yarn test:scripts summarize-link --models ollama:phi3 --out ./temp + run: yarn test:scripts summarize-link --models ollama:phi3 --out ./temp/summarize-link diff --git a/packages/sample/genaisrc/rag.genai.js b/packages/sample/genaisrc/rag.genai.js index 5a6fa05104..8b97ccb450 100644 --- a/packages/sample/genaisrc/rag.genai.js +++ b/packages/sample/genaisrc/rag.genai.js @@ -1,6 +1,6 @@ script({ title: "rag", - model: "openai:gpt-3.5-turbo", + model: "ollama:phi3", files: "src/rag/*", tests: { files: "src/rag/*", @@ -8,7 +8,7 @@ script({ }, }) -$`You are a helpful assistant. Summarize the files in MARKDOWN, PDF, WORD and ALL.` +$`Summarize MARKDOWN, PDF, WORD and ALL. Use one short sentence.` const embeddingsModel = env.vars.embeddingsModel || "ollama:nomic-embed-text" From b9c30df72133030c4fded8e03193e6c553cf2f91 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Fri, 16 Aug 2024 12:26:53 -0700 Subject: [PATCH 5/5] move test to ollama --- packages/sample/genaisrc/summary-of-summary-phi3.genai.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sample/genaisrc/summary-of-summary-phi3.genai.js b/packages/sample/genaisrc/summary-of-summary-phi3.genai.js index 9781464108..c7a231226f 100644 --- a/packages/sample/genaisrc/summary-of-summary-phi3.genai.js +++ b/packages/sample/genaisrc/summary-of-summary-phi3.genai.js @@ -1,8 +1,9 @@ script({ - model: "openai:gpt-4-32k", + model: "ollama:phi3", title: "summary of summary - phi3", + files: ["src/rag/*.md"], tests: { - files: ["src/rag/*"], + files: ["src/rag/*.md"], keywords: ["markdown", "lorem", "microsoft"], } }) @@ -19,4 +20,4 @@ for (const file of env.files) { def("FILE", { ...file, content: text }) } // use summary -$`Summarize all the FILE.` +$`Summarize FILE with short sentence.`