cache

packages/core/src/chatcache.ts

@@ -20,11 +20,9 @@
     ChatCompletationRequestCacheValue
 >
 
-export function getChatCompletionCache(
-    name?: string
-): ChatCompletationRequestCache {
+export function getChatCompletionCache(): ChatCompletationRequestCache {
     return JSONLineCache.byName<
         ChatCompletionRequestCacheKey,
         ChatCompletationRequestCacheValue
-    >(name || CHAT_CACHE)
+    >(CHAT_CACHE)
 }

packages/core/src/openai.ts

@@ -57,7 +57,6 @@
         maxCachedTemperature = MAX_CACHED_TEMPERATURE,
         maxCachedTopP = MAX_CACHED_TOP_P,
         cache: useCache,
-        cacheName,
         retry,
         retryDelay,
         maxDelay,
@@ -68,7 +67,7 @@
     const { model } = parseModelIdentifier(req.model)
     const encoder = await resolveTokenEncoder(model)
 
-    const cache = getChatCompletionCache(cacheName)
+    const cache = getChatCompletionCache()
     const caching =
         useCache === true || // always use cache
         (useCache !== false && // never use cache

packages/core/src/promptrunner.ts

@@ -1,7 +1,7 @@
 import { executeChatSession, tracePromptResult } from "./chat"
 import { Project, PromptScript } from "./ast"
 import { stringToPos } from "./parser"
-import { arrayify, assert, logVerbose, relativePath } from "./util"
+import { arrayify, assert, logError, logVerbose, relativePath } from "./util"
 import { runtimeHost } from "./host"
 import { applyLLMDiff, applyLLMPatch, parseLLMDiffs } from "./diff"
 import { MarkdownTrace } from "./trace"
@@ -327,6 +327,7 @@
                     if (oannotations) annotations = oannotations.slice(0)
                 }
             } catch (e) {
+                logError(e)
                 trace.error(`output processor failed`, e)
             } finally {
                 trace.endDetails()

packages/core/src/types/prompt_template.d.ts

@@ -618,7 +618,7 @@
 
 type MakeOptional<T, P extends keyof T> = Partial<Pick<T, P>> & Omit<T, P>
 
-type PromptArgs = Omit<PromptScript, "text" | "id" | "jsSource" | "activation">
+type PromptArgs = Omit<PromptScript, "text" | "id" | "jsSource" | "activation", "cacheName">
 
 type PromptSystemArgs = Omit<
     PromptArgs,

packages/sample/genaisrc/cache.genai.mts

@@ -1,6 +1,8 @@
 script({
     model: "openai:gpt-3.5-turbo",
-    tests: {},
+    cache: true,
+    cacheName: "gpt-cache",
+    tests: [{}, {}], // run twice to trigger caching
 })
 
 const cache = await workspace.cache<number, number>("test-cache")
@@ -15,3 +17,21 @@ const values = await cache.values()
 if (!values.includes(value)) throw new Error(`unexpected values: ${values}`)
 
 console.log(`cache test passed`)
+
+$`Generate a random word.`
+
+defOutputProcessor(async ({ text }) => {
+    console.error(`process output`)
+    const pcache = await workspace.cache<string, string>("poem-cache")
+    const cached = await pcache.get("poem-result")
+    if (cached) {
+        console.error(`cache hit ${cached} | ${text}`)
+        if (cached !== text) {
+            console.error(`cached value mismatch`)
+            throw new Error(`unexpected cached value: ${cached}`)
+        }
+    } else {
+        console.error(`storing poem in cache`)
+        await pcache.set("poem-result", text)
+    }
+})