pull request describe guide (#508)

* add guide on pull requests

* add workflow

* tune comments

* pr comment

* pr comments

* more tuning

.github/workflows/genai-pr-commit-review.yml

@@ -11,8 +11,6 @@ jobs:
     build:
         runs-on: ubuntu-latest
         permissions:
-            actions: read
-            contents: write
             pull-requests: write
         steps:
             - uses: actions/checkout@v4

.github/workflows/genai-pr-docs-commit-review.yml

@@ -0,0 +1,43 @@
+name: genai pull request docs commit review
+on:
+    pull_request:
+        paths:
+            - docs/**/*.md
+            - docs/**/*.mdx
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        permissions:
+            pull-requests: write
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: "recursive"
+                  fetch-depth: 10
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "20"
+                  cache: yarn
+            - run: yarn install --frozen-lockfile
+            - name: cache .genaiscript
+              uses: actions/cache@v4
+              with:
+                  path: .genaiscript/cache
+                  key: genaiscript-${{ hashFiles('**/yarn.lock') }}
+            - name: compile
+              run: yarn compile
+            - name: git stuff
+              run: git fetch origin && git pull origin main:main
+            - name: genaiscript pr-review-commit
+              run: node packages/cli/built/genaiscript.cjs run pr-docs-review-commit --out ./temp/genai/pr-docs-review-commit -prr --out-trace $GITHUB_STEP_SUMMARY
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha}}
+                  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+                  OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
+                  OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+            - name: Archive genai results
+              uses: actions/upload-artifact@v4
+              with:
+                  name: genai-results
+                  path: ./temp/genai/**

.github/workflows/genai-pr-review.yml

@@ -12,8 +12,6 @@ jobs:
     build:
         runs-on: ubuntu-latest
         permissions:
-            actions: read
-            contents: write
             pull-requests: write
         steps:
             - uses: actions/checkout@v4

docs/src/content/docs/guides/pull-request-reviewer.mdx

@@ -0,0 +1,106 @@
+---
+title: Pull Request Reviewer
+sidebar:
+  order: 20
+---
+import { Code } from '@astrojs/starlight/components';
+import importedCode from "../../../../../packages/sample/genaisrc/pr-review.genai?raw"
+
+The **pull request reviewer** is a GenAIScript that runs in the context of a pull request. 
+It can be used to review the changes in the pull request and provide feedback to the author. 
+The reviewer can also suggest changes to the code, documentation, or other files in the pull request.
+
+The output of the LLM is inserted as a comment in the pull request conversation 
+(and updated as needed to avoid duplicates). 
+Here is an [example](https://github.com/microsoft/genaiscript/pull/504#issuecomment-2145273728) in the GenAIScript repository.
+
+## Step 1: The script
+
+You can prototype the pull request reviewer script in a branch with known changes so that you can assess
+the quality of the results. As you start using it in your build, you will be able to also refine it later on.
+
+### `git diff`
+
+The script starts by running `git diff` to get the changes in the pull request. Since we also know which folder
+to ignore and which file we care about, we can provide additional filters to git to minimize the generated diff.
+
+```js
+const { stdout: diff } = await host.exec("git", [
+    "diff",
+    "main",
+    "--",
+    "**.ts",
+    ":!**/genaiscript.d.ts", // git exclude format
+    ":!genaisrc/*",
+    ":!.github/*",
+    ":!.vscode/*",
+    ":!yarn.lock",
+])
+```
+
+The diff is then inserted in the prompt using the [def](/genaiscript/reference/scripts/context) function.
+
+```js
+def("GIT_DIFF", diff, {
+    language: "diff",
+    maxTokens: 20000,
+})
+```
+
+### Task
+
+The second part of the prompt consists of creating the task and the persona for the LLM.
+
+```js
+$`You are an expert software developer and architect. You are
+an expert in software reliability, security, scalability, and performance.
+
+GIT_DIFF contains the changes the pull request branch. Analyze the changes in GIT_DIFF in your mind.
+
+If the changes look good, respond "LGTM :rocket:". If you have any concerns, provide a brief description of the concerns.
+`
+```
+
+Since we are reviewing TypeScript, we also pre-load the system prompt that prepares the TypeScript mindset of the LLM.
+
+```js '"system.typescript"'
+script({
+    ...,
+    system: [
+        "system",
+        "system.typescript",
+    ],
+})
+```
+
+### Access to the file system
+
+The diff is a partial view of the files and the LLM needs to access the full content of the files 
+to provide a meaningful review. To enable this scenario, 
+
+```js 'tools: ["fs_find_files", "fs_read_file"]'
+script({
+    ...,
+    tools: ["fs_find_files", "fs_read_file"],
+})
+```
+
+### All together
+
+<Code code={importedCode} wrap={true} lang="js" title="pr-review.genai.js" />
+
+## Step 2: Automation in Github Actions
+
+Add this step to your Github Actions workflow to automate the pull request review process.
+
+```yaml
+permissions:
+    pull-requests: write # permission to write a comment
+
+...
+
+    - run: npx --yes genaiscript run ... --out ./temp/genai/pr-review -prc --out-trace $GITHUB_STEP_SUMMARY
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        ... # LLM secrets
+```

packages/core/src/liner.ts

@@ -1,8 +1,10 @@
 /**
- * @param text Adds 1-based line numbrers
- * @returns 
+ * @param text Adds 1-based line numbers
+ * @returns
  */
-export function addLineNumbers(text: string) {
+export function addLineNumbers(text: string, language?: string) {
+    if (language === "diff") return text
+
     return text
         .split("\n")
         .map((line, i) => `[${i + 1}] ${line}`)

packages/core/src/promptdom.ts

@@ -129,9 +129,9 @@ function renderDefNode(def: PromptDefNode): string {
         language === "markdown" || language === "mdx"
             ? MARKDOWN_PROMPT_FENCE
             : PROMPT_FENCE
-    const norm = (s: string) => {
+    const norm = (s: string, lang: string) => {
         s = (s || "").replace(/\n*$/, "")
-        if (s && lineNumbers) s = addLineNumbers(s)
+        if (s && lineNumbers) s = addLineNumbers(s, lang)
         if (s) s += "\n"
         return s
     }
@@ -149,7 +149,7 @@ function renderDefNode(def: PromptDefNode): string {
             dfence = ""
         }
     }
-    body = norm(body)
+    body = norm(body, dtype)
     while (dfence && body.includes(dfence)) {
         dfence += "`"
     }

packages/sample/genaisrc/pr-docs-review-commit.genai.js

@@ -0,0 +1,35 @@
+script({
+    model: "openai:gpt-4",
+    files: [],
+    title: "pull request docs review",
+    system: ["system", "system.technical", "system.annotations"],
+    tools: ["fs_find_files", "fs_read_file"],
+})
+
+const { stdout: diff } = await host.exec("git", [
+    "diff",
+    "main",
+    "--",
+    "docs/**.md",
+    "docs/**.mdx",
+])
+
+def("GIT_DIFF", diff, {
+    language: "diff",
+    maxTokens: 20000,
+    lineNumbers: false,
+})
+
+$`You are an expert technical documentation writer.
+
+GIT_DIFF contains the changes the current branch.
+Analyze the changes in GIT_DIFF in your mind and provide feedback on the documentation.
+
+- the content is Markdown or MDX to be rendered with Astro Starlight https://starlight.astro.build/
+- ignore all whitespace issues
+- ignore code and imports in .mdx files
+- ignore '...' ellipsis errors in code snippets. This placeholder is perfectly acceptable in code snippets.
+- ignore capitalization errors
+- read the full source code of the files if you need more context
+
+`

packages/sample/genaisrc/pr-review.genai.js

@@ -5,23 +5,27 @@ script({
     system: [
         "system",
         "system.typescript",
-        "system.fs_find_files",
-        "system.fs_read_file",
     ],
+    tools: ["fs_find_files", "fs_read_file"],
 })
 
-const { stdout: changes } = await host.exec("git", [
+const { stdout: diff } = await host.exec("git", [
     "diff",
     "main",
     "--",
+    "**.ts",
     ":!**/genaiscript.d.ts",
     ":!genaisrc/*",
     ":!.github/*",
     ":!.vscode/*",
     ":!yarn.lock",
 ])
 
-def("GIT_DIFF", changes, { language: "diff", maxTokens: 20000, lineNumbers: false })
+def("GIT_DIFF", diff, {
+    language: "diff",
+    maxTokens: 20000,
+    lineNumbers: false,
+})
 
 $`You are an expert software developer and architect. You are
 an expert in software reliability, security, scalability, and performance.
@@ -30,13 +34,12 @@ an expert in software reliability, security, scalability, and performance.
 
 GIT_DIFF contains the changes the pull request branch.
 
-Provide a high level review of the changes in the pull request. Do not enter into details.
+Analyze the changes in GIT_DIFF in your mind.
 
 If the changes look good, respond "LGTM :rocket:". If you have any concerns, provide a brief description of the concerns.
 
 - All the TypeScript files are compiled and type-checked by the TypeScript compiler. Do not report issues that the TypeScript compiler would find.
 - only report functional issues
 - Use emojis
 - If available, suggest code fixes and improvements
-
 `