microsoft · pelikhan · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024 · Jun 4, 2024
diff --git a/.github/workflows/genai-pr-commit-review.yml b/.github/workflows/genai-pr-commit-review.yml
@@ -27,7 +27,7 @@ jobs:
             - name: cache .genaiscript
               uses: actions/cache@v4
               with:
-                  path: .genaiscript
+                  path: .genaiscript/cache
                   key: genaiscript-${{ hashFiles('**/yarn.lock') }}
             - name: compile
               run: yarn compile

diff --git a/.github/workflows/genai-pr-review.yml b/.github/workflows/genai-pr-review.yml
@@ -1,7 +1,7 @@
 name: genai pull request review
 on:
     pull_request:
-        types: [review_requested, ready_for_review]
+        types: [ready_for_review]
         paths:
             - yarn.lock
             - ".github/workflows/ollama.yml"
@@ -28,21 +28,21 @@ jobs:
             - name: cache .genaiscript
               uses: actions/cache@v4
               with:
-                  path: .genaiscript
+                  path: .genaiscript/cache
                   key: genaiscript-${{ hashFiles('**/yarn.lock') }}
             - name: compile
               run: yarn compile
             - name: git stuff
               run: git fetch origin && git pull origin main:main
             - name: genaiscript pr-describe
-              run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp/genai/pr-describe -prd pr-describe --out-trace $GITHUB_STEP_SUMMARY
+              run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp/genai/pr-describe -prd --out-trace $GITHUB_STEP_SUMMARY
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
                   OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
                   OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
             - name: genaiscript pr-review
-              run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp/genai/pr-review -prc pr-review --out-trace $GITHUB_STEP_SUMMARY
+              run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp/genai/pr-review -prc --out-trace $GITHUB_STEP_SUMMARY
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   GITHUB_COMMIT_SHA: ${{ github.event.pull_request.base.sha}}

diff --git a/docs/src/content/docs/getting-started/automating-scripts.mdx b/docs/src/content/docs/getting-started/automating-scripts.mdx
@@ -7,6 +7,8 @@ keywords: automation, CLI, batch processing, CI/CD integration, script execution
 ---
 import { FileTree } from '@astrojs/starlight/components';
 import { Image } from 'astro:assets';
+import { Code } from "@astrojs/starlight/components"
+import prDescribeSrc from "../../../../../packages/sample/genaisrc/pr-describe.genai.js?raw"
 
 Once you have a script that you are happy with, you can automate it using the [command line interface](/genaiscript/reference/cli).
 
@@ -29,6 +31,9 @@ Add `--yes` flag skips the confirmation prompt.
 
 :::
 
+You can use the CLI to run your scripts in a CI/CD pipeline. 
+The CLI will return a non-zero exit code if the script fails, which can be used to fail the pipeline.
+
 ### Apply Edits
 
 Add the `--apply-edits` flag to the CLI to automatically write the file edits.
@@ -41,28 +46,143 @@ npx --yes genaiscript run <script> <...files> --apply-edits
 
 An LLM maybe generate arbitrary code that can be harmful to your system. 
 We recommend that you review the modified code before executing it. This could be done through a separate
-branch and a pull request.
+branch and a pull request. You can also use a [container](/genaiscript/reference/scripts/container)
+to run the script in a sandboxed environment.
 
 Refer to [Security and Trust](/genaiscript/reference/security-and-trust) for more discussion.
 
 :::
 
 
-## Batch run scripts
+## GitHub Action
+
+[GitHub Actions](https://docs.github.com/en/actions) is a continuous integration and continuous delivery (CI/CD) 
+platform that allows you to automate your build, test, and deployment pipeline. 
+This section integrates how to integrate your genai scripts in workflows and pull requests.
+
+### Authentication
+
+Configure the [secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) 
+and [variables](https://docs.github.com/en/actions/learn-github-actions/variables)
+on your repository or organization so that GenAIScript can connect to your LLM.
 
-If you want to apply a GenAIScript script individually to a list of files, you can use the [batch](/genaiscript/reference/cli/batch) command.
+The secrets and variables should match the `.env` file in your local environment.
 
-```sh wrap "batch"
-npx --yes genaiscript batch <script> <...files>
+### Running a script
+
+Use the [cli](/genaiscript/reference/cli/run/) to run the script in a GitHub Action. 
+Make sure to pass the secrets and variables to the script.
+
+```yaml
+    - run: npx --yes genaiscript run <script> <...files>
+        env:
+            # variables
+            OPENAI_API_TYPE: ${{ env.OPENAI_API_TYPE }}
+            OPENAI_API_BASE: ${{ env.OPENAI_API_BASE }}
+            # secret, redacted
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 ```
 
-In this case, the script will be applied to each file in the list, and the output will be stored in an output folder.
+### Caching LLM queries
 
-## CI/CD
+GenAIScript stores LLM request caches in the `.genaiscript/cache` folder.
+You can use the [actions/cache](https://github.com/actions/cache) action to cache the LLM queries.
 
-You can use the CLI to run your scripts in a CI/CD pipeline. 
-The CLI will return a non-zero exit code if the script fails, which can be used to fail the pipeline.
+```yaml
+    - name: cache genaiscript
+        uses: actions/cache@v4
+        with:
+            path: .genaiscript/cache
+            key: genaiscript
+```
+
+### Add the trace to the action summary
+
+Use the `out-trace` flag to output the trace to the summary file, `$GITHUB_STEP_SUMMARY`
+(see [example](https://github.com/microsoft/genaiscript/actions/runs/9370477073#summary-25797526178)).
+
+```yaml "--out-trace $GITHUB_STEP_SUMMARY"
+- run: npx --yes genaiscript run ... --out-trace $GITHUB_STEP_SUMMARY
+```
+
+### Annotations
+
+You can use [annotations](/genaiscript/reference/scripts/annotations) to automatically annotate the 
+last commit with errors and warnings.
+
+## GitHub Pull request
+
+If your GitHub Action is triggered by a [pull request event](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request),
+you can use the following flags to add comments: description, conversation and reviews.
+
+In order to create comments, the workflow must have the `pull-requests: write` [permission](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs).
+
+```yaml
+permissions:
+    pull-requests: write
+```
+
+### Diff
+
+You can use `host.exec` to execute [git]() command to retrieve information about the current pull request.
+
+```js
+const { stdout: changes } = await host.exec("git", [
+    "diff",
+    "main",
+    "--",
+    ":!**/genaiscript.d.ts",
+    ":!genaisrc/*",
+    ":!.github/*",
+    ":!.vscode/*",
+    ":!yarn.lock",
+])
+
+def("GIT_DIFF", changes, {
+    language: "diff",
+    maxTokens: 20000,
+    lineNumbers: false,
+})
+```
+
+### Update Description
+
+The `--pull-request-description` inserts the LLM output into the pull request section (see [example](https://github.com/microsoft/genaiscript/pull/504)).
+The command takes an optional string argument to uniquely identify this comment, it is used to update the comment (default is the script id).
+
+```yaml "--pull-request-description"
+- run: npx --yes genaiscript run --pull-request-description
+```
+
+If you want to run this script when the pull request is ready for review, use the [`ready_for_review`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) event.
+
+```yaml
+on:
+    pull_request:
+        types: [ready_for_review]
+```
+
+:::note
 
-- use [secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) in GitHub Actions to populate the environment variables with secrets.
-- use [annotations](/genaiscript/reference/scripts/annotations) to generate SARIF files 
-that can be uploaded to GitHub Actions as security reports.
+The comment is enclosed in two XML comments (`<genaiscript begin [id]>`, `<genaiscript end [id]>`) 
+to allow for easy identification and removal. Make sure to keep those.
+
+:::
+
+### Conversation comment
+
+The `--pull-request-comment` adds the LLM output as a comment to the pull request conversation (see [example](https://github.com/microsoft/genaiscript/pull/504#issuecomment-2145273728)).
+The optional argument is an identifier for the comment (default is the script id).
+
+```yaml "--pull-request-comment"
+- run: npx --yes genaiscript run --pull-request-comment
+```
+
+### Review comments
+
+Use the `--pull-request-reviews` to convert [annotations](/genaiscript/reference/scripts/annotations) as review comments 
+**to the last commit** on the pull request (see [example](https://github.com/microsoft/genaiscript/pull/504#pullrequestreview-2093960791)).
+
+```yaml "--pull-request-reviews"
+- run: npx --yes genaiscript run --pull-request-reviews
+```
diff --git a/docs/src/content/docs/getting-started/testing-scripts.mdx b/docs/src/content/docs/getting-started/testing-scripts.mdx
@@ -7,8 +7,6 @@ keywords: testing, scripts, validation, GenAIScript CLI, automation
 ---
 
 import { Image } from "astro:assets"
-import providerSrc from "../../../../../packages/core/src/genaiscript-api-provider.mjs?raw"
-import { Code } from "@astrojs/starlight/components"
 import testExplorerSrc from "../../../assets/vscode-test-explorer.png"
 import testExplorerAlt from "../../../assets/vscode-test-explorer.png.txt?raw"
 

diff --git a/docs/src/content/docs/guides/sharing-scripts.mdx b/docs/src/content/docs/guides/sharing-scripts.mdx
@@ -1,5 +1,7 @@
 ---
 title: Sharing scripts
+description: Learn how to share GenAIScript scripts across projects using Git repositories, submodules, and GitHub Gists.
+keywords: sharing scripts, git submodules, GitHub Gists, script management, code collaboration
 ---
 import { FileTree } from "@astrojs/starlight/components"
 

diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -22,8 +22,8 @@ Options:
   -od, --out-data <string>                   output file for data (.jsonl/ndjson will be aggregated). JSON schema information and validation will be included if available.
   -oa, --out-annotations <string>            output file for annotations (.csv will be rendered as csv, .jsonl/ndjson will be aggregated)
   -ocl, --out-changelog <string>             output file for changelogs
-  -prc, --pull-request-comment <string>      create comment on a pull request.
-  -prd, --pull-request-description <string>  upsert comment on a pull request description.
+  -prc, --pull-request-comment [string]      create comment on a pull request. (default: "")
+  -prd, --pull-request-description [string]  upsert comment on a pull request description. (default: "")
   -prr, --pull-request-reviews               create pull request reviews from annotations
   -j, --json                                 emit full JSON response to output
   -y, --yaml                                 emit full YAML response to output
@@ -115,7 +115,7 @@ Options:
   -td, --test-delay <string>          delay between tests in seconds
   --no-cache                          disable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  propmtfoo version, default is ^0.60.0
+  -pv, --promptfoo-version [version]  propmtfoo version, default is ^0.61.0
   -os, --out-summary <file>           append output summary in file
   -h, --help                          display help for command
 ```

diff --git a/docs/src/content/docs/reference/cli/index.mdx b/docs/src/content/docs/reference/cli/index.mdx
@@ -10,7 +10,7 @@ import { Steps } from "@astrojs/starlight/components"
 import { Tabs, TabItem } from "@astrojs/starlight/components"
 
 The GenAIScript CLI **`genaiscript`** runs GenAIScript scripts
-outside of Visual Studio.
+outside of Visual Studio and in your [automation](/genaiscript/getting-starting/automating-scripts).
 
 ```sh
 npx --yes genaiscript ...

diff --git a/docs/src/content/docs/reference/scripts/annotations.md b/docs/src/content/docs/reference/scripts/annotations.md
@@ -35,6 +35,14 @@ the precision of the LLM's responses and reduces the likelihood of hallucination
 By default, the annotations use the [GitHub Action Commands](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message) syntax.
 This means that the annotations will automatically be extracted by GitHub if you run your script in a GitHub Action.
 
+## Github Pull Request Review Comments
+
+Use the `--pull-request-review-comment` flag on the [cli](/genaiscript/reference/cli/run/) to add annotations as review comments on a pull request.
+
+```sh "cli"
+npx --yes genaiscript run ... --pull-request-review-comment
+```
+
 ## Visual Studio Code Programs
 
 Annotations are converted into Visual Studio **Diagnostics**, which are presented to the user
@@ -53,31 +61,31 @@ name: "Upload SARIF"
 # Run workflow each time code is pushed to your repository and on a schedule.
 # The scheduled workflow runs every Thursday at 15:45 UTC.
 on:
-  push:
-  schedule:
-    - cron: '45 15 * * 4'
+    push:
+    schedule:
+        - cron: "45 15 * * 4"
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    permissions:
-      # required for all workflows
-      security-events: write
-      # only required for workflows in private repositories
-      actions: read
-      contents: read
-    steps:
-      # This step checks out a copy of your repository.
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      # Run GenAIScript tools
-      - name: Run GenAIScript
-        run: npx --yes genaiscript ... -oa result.sarif
-      # Upload the generated SARIF file to GitHub
-      - name: Upload SARIF file
-        if: success() || failure()
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: result.sarif
+    build:
+        runs-on: ubuntu-latest
+        permissions:
+            # required for all workflows
+            security-events: write
+            # only required for workflows in private repositories
+            actions: read
+            contents: read
+        steps:
+            # This step checks out a copy of your repository.
+            - name: Checkout repository
+              uses: actions/checkout@v4
+            # Run GenAIScript tools
+            - name: Run GenAIScript
+              run: npx --yes genaiscript ... -oa result.sarif
+            # Upload the generated SARIF file to GitHub
+            - name: Upload SARIF file
+              if: success() || failure()
+              uses: github/codeql-action/upload-sarif@v3
+              with:
+                  sarif_file: result.sarif
 ```
 
 ### Limitations

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -24,7 +24,7 @@
   "license": "MIT",
   "dependencies": {
     "pdfjs-dist": "4.3.136",
-    "promptfoo": "^0.60.0",
+    "promptfoo": "^0.61.0",
     "dockerode": "^4.0.2",
     "tree-sitter-wasms": "^0.1.11",
     "typescript": "5.4.5",
@@ -42,7 +42,7 @@
     "@types/dockerode": "^3.3.29",
     "@types/fs-extra": "^11.0.4",
     "@types/memorystream": "^0.3.4",
-    "@types/node": "^20.12.12",
+    "@types/node": "^20.14.1",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.2",
     "@types/prompts": "^2.4.9",
@@ -61,13 +61,13 @@
     "mammoth": "^1.7.2",
     "memorystream": "^0.3.1",
     "node-sarif-builder": "^3.1.0",
-    "openai": "^4.47.2",
+    "openai": "^4.47.3",
     "ora": "^8.0.1",
     "pretty-bytes": "^6.1.1",
     "prompts": "^2.4.2",
     "replace-ext": "^2.0.0",
     "semver": "^7.6.2",
-    "tsx": "^4.11.0",
+    "tsx": "^4.11.2",
     "zx": "^8.1.2"
   },
   "scripts": {

diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -101,12 +101,12 @@ export async function cli() {
         )
         .option("-ocl, --out-changelog <string>", "output file for changelogs")
         .option(
-            "-prc, --pull-request-comment <string>",
-            "create comment on a pull request."
+            "-prc, --pull-request-comment [string]",
+            "create comment on a pull request.", ""
         )
         .option(
-            "-prd, --pull-request-description <string>",
-            "upsert comment on a pull request description."
+            "-prd, --pull-request-description [string]",
+            "upsert comment on a pull request description.", ""
         )
         .option(
             "-prr, --pull-request-reviews",