From 86d67bb671bf25a9c839cd1d4ee84a892e4540ff Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 5 Jun 2024 01:13:18 +0000 Subject: [PATCH 1/4] add prr cache --- packages/cli/src/cli.ts | 4 ++ packages/cli/src/run.ts | 22 ++++++++++- packages/core/src/cache.ts | 6 +-- packages/core/src/chat.ts | 9 +++-- packages/core/src/constants.ts | 6 ++- packages/core/src/github.ts | 58 +++++++++++++++++++++++++--- packages/vscode/src/airequesttree.ts | 4 +- packages/vscode/src/state.ts | 7 ++-- 8 files changed, 96 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 8be4ee0a59..ab76fa5fd0 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -112,6 +112,10 @@ export async function cli() { "-prr, --pull-request-reviews", "create pull request reviews from annotations" ) + .option( + "--no-pull-request-reviews-cache", + "disable pull request reviews cache" + ) .option("-j, --json", "emit full JSON response to output") .option("-y, --yaml", "emit full YAML response to output") .option( diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 890a2d1f52..e621fa1d49 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -35,6 +35,10 @@ import { githubUpsetPullRequest, githubCreatePullRequestReviews, githubCreateIssueComment, + GITHUB_PULL_REQUEST_REVIEWS_CACHE, + JSONLineCache, + PullRequestReviewsCacheKey, + PullRequestReviewsCacheValue, } from "genaiscript-core" import { capitalize } from "inflection" import { basename, resolve, join, relative } from "node:path" @@ -62,6 +66,7 @@ export async function runScript( pullRequestComment: string | boolean pullRequestDescription: string | boolean pullRequestReviews: boolean + pullRequestReviewsCache: boolean outData: string label: string temperature: string @@ -101,6 +106,7 @@ export async function runScript( const maxTokens = normalizeInt(options.maxTokens) const maxToolCalls = normalizeInt(options.maxToolCalls) const cache = !!options.cache + const pullRequestReviewsCache = !!options.pullRequestReviewsCache const applyEdits = !!options.applyEdits const csvSeparator = options.csvSeparator || "\t" const removeOut = options.removeOut @@ -368,8 +374,20 @@ ${Array.from(files) if (pullRequestReviews && res.annotations?.length) { const info = parseGHTokenFromEnv(process.env) - if (info.repository && info.issue) - await githubCreatePullRequestReviews(script, info, res.annotations) + if (info.repository && info.issue) { + const cache = pullRequestReviewsCache + ? JSONLineCache.byName< + PullRequestReviewsCacheKey, + PullRequestReviewsCacheValue + >(GITHUB_PULL_REQUEST_REVIEWS_CACHE) + : undefined + await githubCreatePullRequestReviews( + script, + info, + res.annotations, + { cache } + ) + } } if (pullRequestComment && res.text) { diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index ee548fda32..2642cc7ba4 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -7,17 +7,17 @@ import { CORE_VERSION } from "./version" export type CacheEntry = { sha: string; key: K; val: V } -export class Cache extends EventTarget { +export class JSONLineCache extends EventTarget { private _entries: Record> private constructor(public readonly name: string) { super() } - static byName(name: string): Cache { + static byName(name: string): JSONLineCache { name = name.replace(/[^a-z0-9_]/gi, "_") const key = "cacheKV." + name if (host.userState[key]) return host.userState[key] - const r = new Cache(name) + const r = new JSONLineCache(name) host.userState[key] = r return r } diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index 2ec3b2406b..4b04da88f8 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -1,5 +1,5 @@ import OpenAI from "openai" -import { Cache } from "./cache" +import { JSONLineCache } from "./cache" import { MarkdownTrace } from "./trace" import { PromptImage } from "./promptdom" import { AICIRequest } from "./aici" @@ -16,6 +16,7 @@ import { import { validateFencesWithSchema, validateJSONWithSchema } from "./schema" import dedent from "ts-dedent" import { + CHAT_CACHE, DEFAULT_MODEL, DEFAULT_TEMPERATURE, MAX_DATA_REPAIRS, @@ -87,7 +88,7 @@ export type ChatCompletationRequestCacheValue = { finishReason: ChatCompletionResponse["finishReason"] } -export type ChatCompletationRequestCache = Cache< +export type ChatCompletationRequestCache = JSONLineCache< ChatCompletionRequestCacheKey, ChatCompletationRequestCacheValue > @@ -95,10 +96,10 @@ export type ChatCompletationRequestCache = Cache< export function getChatCompletionCache( name?: string ): ChatCompletationRequestCache { - return Cache.byName< + return JSONLineCache.byName< ChatCompletionRequestCacheKey, ChatCompletationRequestCacheValue - >(name || "chatv2") + >(name || CHAT_CACHE) } export interface ChatCompletionsProgressReport { diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index d167d4160b..e33de00946 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -170,4 +170,8 @@ export const DOCKER_CONTAINER_VOLUME = "/app" export const CLI_RUN_FILES_FOLDER = "files" export const GITHUB_API_VERSION = "2022-11-28" -export const GITHUB_TOKEN = "GITHUB_TOKEN" \ No newline at end of file +export const GITHUB_TOKEN = "GITHUB_TOKEN" + +export const AI_REQUESTS_CACHE = "airequests" +export const CHAT_CACHE = "chatv2" +export const GITHUB_PULL_REQUEST_REVIEWS_CACHE = "prr" \ No newline at end of file diff --git a/packages/core/src/github.ts b/packages/core/src/github.ts index 7d9a98a24e..da1f296281 100644 --- a/packages/core/src/github.ts +++ b/packages/core/src/github.ts @@ -1,9 +1,15 @@ import { assert } from "node:console" -import { GITHUB_API_VERSION, GITHUB_TOKEN } from "./constants" +import { + GITHUB_API_VERSION, + GITHUB_PULL_REQUEST_REVIEWS_CACHE, + GITHUB_TOKEN, +} from "./constants" import { createFetch } from "./fetch" import { host } from "./host" import { link, prettifyMarkdown } from "./markdown" import { logError, logVerbose, normalizeInt } from "./util" +import { string } from "mathjs" +import { JSONLineCache } from "./cache" export interface GithubConnectionInfo { token: string @@ -211,6 +217,7 @@ async function githubCreatePullRequestReview( ) { assert(token) const { apiUrl, repository, issue, commitSha } = info + const fetch = await createFetch({ retryOn: [] }) const url = `${apiUrl}/repos/${repository}/pulls/${issue}/comments` const body = { @@ -244,13 +251,36 @@ async function githubCreatePullRequestReview( return r } +export interface PullRequestReviewsCacheKey { + repository: string + issue: number + scriptId: string + annotation: Diagnostic +} + +export interface PullRequestReviewsCacheValue { + created: boolean + statusText: string + html_url: string +} + +export type PullRequestReviewsCache = JSONLineCache< + PullRequestReviewsCacheKey, + PullRequestReviewsCacheValue +> + export async function githubCreatePullRequestReviews( script: PromptScript, info: GithubConnectionInfo, - annotations: Diagnostic[] + annotations: Diagnostic[], + options?: { + cache?: PullRequestReviewsCache + } ): Promise { + const { repository, issue, sha } = info + const { cache } = options || {} + if (!annotations?.length) return true - const { issue, sha } = info if (!issue) { logError("missing pull request number") return false @@ -266,7 +296,25 @@ export async function githubCreatePullRequestReviews( } // code annotations - for (const annotation of annotations) - await githubCreatePullRequestReview(script, info, token, annotation) + for (const annotation of annotations) { + const cacheKey = { + repository, + issue, + scriptId: script.id, + annotation, + } + const cached = await cache?.get(cacheKey) + if (cached) + logVerbose("ignore cached pull request review, " + cached.html_url) + else { + const res = await githubCreatePullRequestReview( + script, + info, + token, + annotation + ) + if (res.created) await cache?.set(cacheKey, res) + } + } return true } diff --git a/packages/vscode/src/airequesttree.ts b/packages/vscode/src/airequesttree.ts index a5192deeb2..b0f167dc84 100644 --- a/packages/vscode/src/airequesttree.ts +++ b/packages/vscode/src/airequesttree.ts @@ -5,7 +5,7 @@ import { ExtensionState, } from "./state" import { CACHE_AIREQUEST_PREFIX, CHANGE, CacheEntry } from "genaiscript-core" -import { Cache } from "genaiscript-core" +import { JSONLineCache } from "genaiscript-core" import { infoUri } from "./markdowndocumentprovider" import { toMarkdownString } from "./markdown" @@ -14,7 +14,7 @@ type AIRequestTreeNode = CacheEntry class AIRequestTreeDataProvider implements vscode.TreeDataProvider { - cache: Cache + cache: JSONLineCache constructor(readonly state: ExtensionState) { this.cache = state.aiRequestCache() this.cache.addEventListener(CHANGE, () => this.refresh()) diff --git a/packages/vscode/src/state.ts b/packages/vscode/src/state.ts index 434ffa4522..2a06be4013 100644 --- a/packages/vscode/src/state.ts +++ b/packages/vscode/src/state.ts @@ -13,7 +13,7 @@ import { isCancelError, delay, CHANGE, - Cache, + JSONLineCache, logInfo, logMeasure, parseAnnotations, @@ -31,6 +31,7 @@ import { fixPromptDefinitions, DEFAULT_MODEL, resolveModelConnectionInfo, + AI_REQUESTS_CACHE, } from "genaiscript-core" import { ExtensionContext } from "vscode" import { VSCodeHost } from "./vshost" @@ -124,7 +125,7 @@ export function snapshotAIRequest(r: AIRequest): AIRequestSnapshot { } function getAIRequestCache() { - return Cache.byName("airequests") + return JSONLineCache.byName(AI_REQUESTS_CACHE) } export class ExtensionState extends EventTarget { @@ -133,7 +134,7 @@ export class ExtensionState extends EventTarget { private _project: Project = undefined private _aiRequest: AIRequest = undefined private _diagColl: vscode.DiagnosticCollection - private _aiRequestCache: Cache = + private _aiRequestCache: JSONLineCache = undefined readonly output: vscode.LogOutputChannel From c995085907f512507cb9e4c9e010df3caddd7e42 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 5 Jun 2024 01:15:27 +0000 Subject: [PATCH 2/4] fix escaping in test --- packages/sample/genaisrc/summarize-link.genai.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sample/genaisrc/summarize-link.genai.js b/packages/sample/genaisrc/summarize-link.genai.js index a9d79ea5fb..4938cbd9cc 100644 --- a/packages/sample/genaisrc/summarize-link.genai.js +++ b/packages/sample/genaisrc/summarize-link.genai.js @@ -7,7 +7,7 @@ script({ temperature: 0, tests: { files: [ - "https://raw.githubusercontent.com/microsoft/genaiscript/main/packages/sample/src/rag/markdown.md", + `https://raw.githubusercontent.com/microsoft/genaiscript/main/packages/sample/src/rag/markdown.md`, ], keywords: "markdown", }, From 9c8b3d400027d3f3419245886bca971d64d6ab55 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 5 Jun 2024 01:19:24 +0000 Subject: [PATCH 3/4] updated docs --- docs/src/content/docs/reference/cli/commands.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index ff19317320..01b7852e68 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -25,6 +25,7 @@ Options: -prc, --pull-request-comment [string] create comment on a pull request. -prd, --pull-request-description [string] upsert comment on a pull request description. -prr, --pull-request-reviews create pull request reviews from annotations + --no-pull-request-reviews-cache disable pull request reviews cache -j, --json emit full JSON response to output -y, --yaml emit full YAML response to output -p, --prompt dry run, don't execute LLM and return expanded prompt From a22fbe8289cecde8686a93d24414107c99ae603e Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 5 Jun 2024 01:22:02 +0000 Subject: [PATCH 4/4] pr feedback --- packages/core/src/github.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/core/src/github.ts b/packages/core/src/github.ts index da1f296281..7532b70c88 100644 --- a/packages/core/src/github.ts +++ b/packages/core/src/github.ts @@ -255,7 +255,9 @@ export interface PullRequestReviewsCacheKey { repository: string issue: number scriptId: string - annotation: Diagnostic + message: string + filename: string + line: number } export interface PullRequestReviewsCacheValue { @@ -301,7 +303,9 @@ export async function githubCreatePullRequestReviews( repository, issue, scriptId: script.id, - annotation, + message: annotation.message, + filename: annotation.filename, + line: annotation.range?.[0]?.[0], } const cached = await cache?.get(cacheKey) if (cached)