From c2ec4dcaccf38a62dadd871942dc8dfd6c521ddc Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 1 Jun 2024 12:29:16 +0000 Subject: [PATCH 001/187] adding cache for .genaiscript --- .github/workflows/genai-pr-describe.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/genai-pr-describe.yml b/.github/workflows/genai-pr-describe.yml index 70b759a1b4..f2e4bcbdd0 100644 --- a/.github/workflows/genai-pr-describe.yml +++ b/.github/workflows/genai-pr-describe.yml @@ -26,6 +26,11 @@ jobs: node-version: "20" cache: yarn - run: yarn install --frozen-lockfile + - name: cache .genaiscript + uses: actions/cache@v4 + with: + path: .genaiscript + key: genaiscript-${{ hashFiles('**/yarn.lock') }} - name: compile run: yarn compile - name: git stuff From 4493a61344ca807ea56a6c04b413795d14552ada Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 1 Jun 2024 12:45:59 +0000 Subject: [PATCH 002/187] better filter in pr-description generator --- genaisrc/pr-describe.genai.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/genaisrc/pr-describe.genai.js b/genaisrc/pr-describe.genai.js index ef7ee0b218..6d85cf9958 100644 --- a/genaisrc/pr-describe.genai.js +++ b/genaisrc/pr-describe.genai.js @@ -9,6 +9,8 @@ const { stdout: changes } = await host.exec("git", [ "main", "--", ":!**/genaiscript.d.ts", + ":!.vscode/*", + ":!yarn.lock", ]) def("GIT_DIFF", changes, { maxTokens: 20000 }) From 047e7030c0bfeec97aec5f71203463640929a19c Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 1 Jun 2024 13:17:34 +0000 Subject: [PATCH 003/187] added pr-review --- .../{genai-pr-describe.yml => genai-pr.yml} | 9 ++++- genaisrc/pr-review.genai.js | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) rename .github/workflows/{genai-pr-describe.yml => genai-pr.yml} (76%) create mode 100644 genaisrc/pr-review.genai.js diff --git a/.github/workflows/genai-pr-describe.yml b/.github/workflows/genai-pr.yml similarity index 76% rename from .github/workflows/genai-pr-describe.yml rename to .github/workflows/genai-pr.yml index f2e4bcbdd0..6b91cc5498 100644 --- a/.github/workflows/genai-pr-describe.yml +++ b/.github/workflows/genai-pr.yml @@ -35,10 +35,17 @@ jobs: run: yarn compile - name: git stuff run: git fetch origin && git pull origin main:main - - name: genaiscript run + - name: genaiscript pr-describe run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp -prd pr-describe env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} + - name: genaiscript pr-review + run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp -prd pr-review + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} + OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} diff --git a/genaisrc/pr-review.genai.js b/genaisrc/pr-review.genai.js new file mode 100644 index 0000000000..b586d750ed --- /dev/null +++ b/genaisrc/pr-review.genai.js @@ -0,0 +1,35 @@ +script({ + model: "openai:gpt-4", + files: [], + title: "pull request review", + system: [ + "system", + "system.typescript", + "system.fs_find_files", + "system.fs_read_file", + "system.annotations" + ], +}) +const { stdout: changes } = await host.exec("git", [ + "diff", + "main", + "--", + ":!**/genaiscript.d.ts", + ":!.vscode/*", + ":!yarn.lock", +]) + +def("GIT_DIFF", changes, { maxTokens: 20000 }) + +$`You are an expert software developer and architect. You are +an expert in software reliability, security, scalability, and performance. + +## Task + +Review the changes in GIT_DIFF and provide feedback to the author using annotations. + +- report errors only, ignore notes and warnings. +- use a friendly tone +- use emojis +- read the full source code of the files if you need more context +` From 313f6e7428e2495e9f0cd70b7a579be6429aa3dc Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Sat, 1 Jun 2024 22:53:17 -0700 Subject: [PATCH 004/187] upload annotations in github pull requests (#501) * upload annotations * put the review in comment, don't comment from ollama * added commit comments * only review .ts files * comment and review with pr-review * consistent logging of gh calls * more tuning of describe * include runid link in generated comment * more logging * typo in run url * fix description slicing * ensure relative path in diagnostics * more logging * and logging * generate pull review comments * renamed option * don't emit start_line if same lines * missing start_side * only consider last commit_id * remove subject_type * store pull request head sha * archive results * tune review * type in change * reduce logging * don't inject line numbers in pr-review-commit * don't log empty errors * don't retry on github resources * more logging * don't inline code snippets * ensure annotations are unique when parsing * fix tracing of diagnostics * only report 3 errors --- .github/workflows/genai-pr.yml | 18 ++- .github/workflows/ollama.yml | 6 +- .vscode/settings.json | 1 + .../content/docs/reference/cli/commands.md | 1 + genaisrc/pr-describe.genai.js | 8 +- genaisrc/pr-review-commit.genai.js | 40 +++++ genaisrc/pr-review.genai.js | 14 +- packages/cli/src/cli.ts | 6 +- packages/cli/src/run.ts | 45 ++---- packages/core/src/annotations.ts | 20 ++- packages/core/src/constants.ts | 3 +- packages/core/src/fetch.ts | 3 + packages/core/src/github.ts | 151 +++++++++++++++--- packages/core/src/markdown.ts | 4 + packages/core/src/promptrunner.ts | 9 +- packages/core/src/util.ts | 18 ++- 16 files changed, 264 insertions(+), 83 deletions(-) create mode 100644 genaisrc/pr-review-commit.genai.js diff --git a/.github/workflows/genai-pr.yml b/.github/workflows/genai-pr.yml index 6b91cc5498..354e97918a 100644 --- a/.github/workflows/genai-pr.yml +++ b/.github/workflows/genai-pr.yml @@ -36,16 +36,30 @@ jobs: - name: git stuff run: git fetch origin && git pull origin main:main - name: genaiscript pr-describe - run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp -prd pr-describe + run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp/genai/pr-describe -prd pr-describe env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} - name: genaiscript pr-review - run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp -prd pr-review + run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp/genai/pr-review -prc pr-review env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_COMMIT_SHA: ${{ github.event.pull_request.base.sha}} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} + - name: genaiscript pr-review-commit + run: node packages/cli/built/genaiscript.cjs run pr-review-commit --out ./temp/genai/pr-review-commit -prr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha}} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }} + OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} + - name: Archive genai results + uses: actions/upload-artifact@v4 + with: + name: genai-results + path: ./temp/genai/** diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml index aa1c28fddc..10d3663b31 100644 --- a/.github/workflows/ollama.yml +++ b/.github/workflows/ollama.yml @@ -17,8 +17,6 @@ on: - "packages/core/**/*" - "packages/cli/**/*" - "packages/samples/**/*" -permissions: - pull-requests: write jobs: tests: runs-on: ubuntu-latest @@ -39,6 +37,4 @@ jobs: - name: download ollama docker run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama - name: run summarize-ollama-phi3 - run: yarn test:summarize --model ollama:phi3 --out ./temp -prc ollama - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: yarn test:summarize --model ollama:phi3 --out ./temp diff --git a/.vscode/settings.json b/.vscode/settings.json index 74a6d16bfc..1937cc9766 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,6 +9,7 @@ "execa", "genai", "Genaiscript", + "genaisrc", "gpspec", "gpspecs", "gptool", diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index 1facfb9fb9..3fb9e481db 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -24,6 +24,7 @@ Options: -ocl, --out-changelog output file for changelogs -prc, --pull-request-comment create comment on a pull request. -prd, --pull-request-description upsert comment on a pull request description. + -prr, --pull-request-reviews create pull request reviews from annotations -j, --json emit full JSON response to output -y, --yaml emit full YAML response to output -p, --prompt dry run, don't execute LLM and return expanded prompt diff --git a/genaisrc/pr-describe.genai.js b/genaisrc/pr-describe.genai.js index 6d85cf9958..7b45036d8e 100644 --- a/genaisrc/pr-describe.genai.js +++ b/genaisrc/pr-describe.genai.js @@ -9,6 +9,8 @@ const { stdout: changes } = await host.exec("git", [ "main", "--", ":!**/genaiscript.d.ts", + ":!genaisrc/*", + ":!.github/*", ":!.vscode/*", ":!yarn.lock", ]) @@ -19,13 +21,13 @@ $`You are an expert software developer and architect. ## Task -- Describe the changes in GIT_DIFF in a way that a software engineer will understand. +- Describe a summary of the changes in GIT_DIFF in a way that a software engineer will understand. ## Instructions - use bullet points to list the changes - use emojis to make the description more engaging -- if needed inline code snippets can be used. The code snippets should be taken -from the changes in GIT_DIFF. +- focus on the most important changes +- ignore comments about imports (like added, remove, changed, etc.) ` diff --git a/genaisrc/pr-review-commit.genai.js b/genaisrc/pr-review-commit.genai.js new file mode 100644 index 0000000000..b514f50c9b --- /dev/null +++ b/genaisrc/pr-review-commit.genai.js @@ -0,0 +1,40 @@ +script({ + model: "openai:gpt-4", + files: [], + title: "pull request commit review", + system: [ + "system", + "system.typescript", + "system.fs_find_files", + "system.fs_read_file", + "system.annotations", + ], +}) +const { stdout: changes } = await host.exec("git", [ + "diff", + "HEAD^", + "HEAD", + "--", + "**.ts", +]) + +def("GIT_DIFF", changes, { maxTokens: 20000, lineNumbers: false }) + +$`You are an expert software developer and architect. You are +an expert in software reliability, security, scalability, and performance. + +## Task + +Review the changes in GIT_DIFF which contains the changes of the last commit in the pull request branch. +Provide feedback to the author using annotations. + +Think step by step and for each annotation explain your result. + +- report 3 most serious errors only, ignore notes and warnings +- only report issues you are absolutely certain about +- do NOT repeat the same issue multiple times +- use a friendly tone +- use emojis +- read the full source code of the files if you need more context +- only report issues about code in GIT_DIFF +` diff --git a/genaisrc/pr-review.genai.js b/genaisrc/pr-review.genai.js index b586d750ed..81b85b2354 100644 --- a/genaisrc/pr-review.genai.js +++ b/genaisrc/pr-review.genai.js @@ -7,14 +7,16 @@ script({ "system.typescript", "system.fs_find_files", "system.fs_read_file", - "system.annotations" ], }) + const { stdout: changes } = await host.exec("git", [ "diff", "main", "--", ":!**/genaiscript.d.ts", + ":!genaisrc/*", + ":!.github/*", ":!.vscode/*", ":!yarn.lock", ]) @@ -26,10 +28,10 @@ an expert in software reliability, security, scalability, and performance. ## Task -Review the changes in GIT_DIFF and provide feedback to the author using annotations. +GIT_DIFF contains the changes the pull request branch. + +Provide a high level review of the changes in the pull request. Do not enter into details. + +If the changes look good, respond LGTM (Looks Good To Me). If you have any concerns, provide a brief description of the concerns. -- report errors only, ignore notes and warnings. -- use a friendly tone -- use emojis -- read the full source code of the files if you need more context ` diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e944f0e516..25a2b7d9d0 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -82,7 +82,7 @@ export async function cli() { program.on("option:quiet", () => setQuiet(true)) program - .command("run") + .command("run", { isDefault: true }) .description("Runs a GenAIScript against files.") .arguments("