refactor: ♻️ replace ephemeral with cacheControl

docs/src/content/docs/reference/scripts/context.md

@@ -203,12 +203,12 @@ def("FILE", env.files, { sliceSample: 100 })
 You can use `cacheControl: "ephemeral"` to specify that the prompt can be cached
 for a short amount of time, and enable prompt caching optimization, which is supported (differently) by various LLM providers.
 
-```js "ephemeral: true"
+```js 'cacheControl("ephemeral")'
 $`...`.cacheControl("ephemeral")
 ```
 
-```js "ephemeral: true"
-def("FILE", env.files, { ephemeral: true })
+```js '"cacheControl: "ephemeral"'
+def("FILE", env.files, { cacheControl: "ephemeral" })
 ```
 
 Read more about [prompt caching](/genaiscript/reference/scripts/prompt-caching).

packages/core/src/anthropic.ts

@@ -27,6 +27,7 @@ import {
     ChatCompletionRequestCacheKey,
     getChatCompletionCache,
 } from "./chatcache"
+import { traceFetchPost } from "./fetch"
 
 const convertFinishReason = (
     stopReason: Anthropic.Message["stop_reason"]
@@ -300,30 +301,31 @@ export const AnthropicChatCompletion: ChatCompletionHandler = async (
     let finishReason: ChatCompletionResponse["finishReason"]
     let usage: ChatCompletionResponse["usage"] | undefined
     const toolCalls: ChatCompletionToolCall[] = []
+    const tools = convertTools(req.tools)
+
+    const mreq = deleteUndefinedValues({
+        model,
+        tools,
+        messages,
+        max_tokens: req.max_tokens || ANTHROPIC_MAX_TOKEN,
+        temperature: req.temperature,
+        top_p: req.top_p,
+        stream: true,
+    })
 
+    trace.detailsFenced("✉️ body", mreq, "json")
     trace.appendContent("\n")
 
     try {
         const messagesApi = caching
             ? anthropic.beta.promptCaching.messages
             : anthropic.messages
-        const stream = messagesApi.stream({
-            model,
-            tools: convertTools(req.tools),
-            messages,
-            max_tokens: req.max_tokens || ANTHROPIC_MAX_TOKEN,
-            temperature: req.temperature,
-            top_p: req.top_p,
-            stream: true,
-            ...headers,
-        })
-
+        const stream = messagesApi.stream({ ...mreq, ...headers })
         for await (const chunk of stream) {
             if (cancellationToken?.isCancellationRequested) {
                 finishReason = "cancel"
                 break
             }
-
             switch (chunk.type) {
                 case "message_start":
                     usage = convertUsage(

packages/core/src/promptdom.ts

@@ -479,7 +479,9 @@ export function createDefData(
     options?: DefDataOptions
 ) {
     if (data === undefined) return undefined
-    let { format, headers, priority, ephemeral } = options || {}
+    let { format, headers, priority, cacheControl } = options || {}
+    cacheControl =
+        cacheControl ?? (options?.ephemeral ? "ephemeral" : undefined)
     if (
         !format &&
         Array.isArray(data) &&
@@ -513,7 +515,10 @@ ${trimNewlines(text)}
 ${trimNewlines(text)}
 `
     // TODO maxTokens does not work well with data
-    return createTextNode(value, { priority, ephemeral })
+    return createTextNode(value, {
+        priority,
+        ephemeral: cacheControl === "ephemeral",
+    })
 }
 
 // Function to append a child node to a parent node.

packages/core/src/types/prompt_template.d.ts

@@ -897,6 +897,8 @@ interface FenceOptions extends LineNumberingOptions, FenceFormatOptions {
     schema?: string
 }
 
+type PromptCacheControlType = "ephemeral"
+
 interface ContextExpansionOptions {
     /**
      * Specifies an maximum of estimated tokens for this entry; after which it will be truncated.
@@ -916,9 +918,14 @@ interface ContextExpansionOptions {
     flex?: number
 
     /**
-     * This text is likely to change and will probably break the prefix cache.
+     * @deprecated use cacheControl instead
      */
     ephemeral?: boolean
+
+    /**
+     * Caching policy for this text. `ephemeral` means the prefix can be cached for a short amount of time.
+     */
+    cacheControl?: PromptCacheControlType
 }
 
 interface RangeOptions {

packages/sample/genaisrc/summarize-cached.genai.mjs

@@ -9,7 +9,7 @@ script({
     ],
 })
 
-def("FILE", env.files, { ephemeral: true })
+def("FILE", env.files, { cacheControl: "ephemeral" })
 
 $`
 Summarize all files in FILE in a single paragraph.