native glob api (#497)

* native glob api

* missing comma

* add pr-describe task

* updated pr-describer

* updated permissions

* comment delete log

* fetch main

* extract proper comment

* more yaml fun

* more git stuff

* fix delete url

* use bullet points

* renamed cli option

* fix delurl

* fix some logging

.github/workflows/genai-pr-describe.yml

@@ -0,0 +1,39 @@
+name: genai pull request describe
+on:
+    pull_request:
+        paths:
+            - yarn.lock
+            - ".github/workflows/ollama.yml"
+            - "packages/core/**/*"
+            - "packages/cli/**/*"
+            - "packages/samples/**/*"
+permissions:
+    pull-requests: write
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        permissions:
+            actions: read
+            contents: write
+            pull-requests: write
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: "recursive"
+                  fetch-depth: 10
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "20"
+                  cache: yarn
+            - run: yarn install --frozen-lockfile
+            - name: compile
+              run: yarn compile
+            - name: git stuff
+              run: git fetch origin && git pull origin main:main
+            - name: genaiscript run
+              run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp -prc pr-describe
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+                  OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
+                  OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}

.github/workflows/ollama.yml

@@ -19,7 +19,6 @@ on:
             - "packages/samples/**/*"
 permissions:
     pull-requests: write
-    contents: read
 jobs:
     tests:
         runs-on: ubuntu-latest
@@ -40,6 +39,6 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run summarize-ollama-phi3
-              run: yarn test:summarize --model ollama:phi3 --out ./temp -ghc ollama
+              run: yarn test:summarize --model ollama:phi3 --out ./temp -prc ollama
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/src/content/docs/reference/cli/commands.md

@@ -15,46 +15,34 @@ Usage: genaiscript run [options] <script> [files...]
 Runs a GenAIScript against files.
 
 Options:
-  -ef, --excluded-files <string...>  excluded files
-  -o, --out <string>                 output folder. Extra markdown fields for
-                                     output and trace will also be generated
-  -rmo, --remove-out                 remove output folder if it exists
-  -ot, --out-trace <string>          output file for trace
-  -od, --out-data <string>           output file for data (.jsonl/ndjson will
-                                     be aggregated). JSON schema information
-                                     and validation will be included if
-                                     available.
-  -oa, --out-annotations <string>    output file for annotations (.csv will be
-                                     rendered as csv, .jsonl/ndjson will be
-                                     aggregated)
-  -ocl, --out-changelog <string>     output file for changelogs
-  -ghc, --github-comment [string]    create github comment on a pull request or
-                                     commit with output. Use commentid to
-                                     upsert comment. (default: "none")
-  -j, --json                         emit full JSON response to output
-  -y, --yaml                         emit full YAML response to output
-  -p, --prompt                       dry run, don't execute LLM and return
-                                     expanded prompt
-  -fe, --fail-on-errors              fails on detected annotation error
-  -r, --retry <number>               number of retries (default: "8")
-  -rd, --retry-delay <number>        minimum delay between retries (default:
-                                     "15000")
-  -md, --max-delay <number>          maximum delay between retries (default:
-                                     "180000")
-  -l, --label <string>               label for the run
-  -t, --temperature <number>         temperature for the run
-  -tp, --top-p <number>              top-p for the run
-  -m, --model <string>               model for the run
-  -mt, --max-tokens <number>         maximum tokens for the run
-  -mtc, --max-tool-calls <number>    maximum tool calls for the run
-  -se, --seed <number>               seed for the run
-  --no-cache                         disable LLM result cache
-  -cn, --cache-name <name>           custom cache file name
-  --cs, --csv-separator <string>     csv separator (default: "\t")
-  -ae, --apply-edits                 apply file edits
-  --vars <namevalue...>              variables, as name=value, stored in
-                                     env.vars
-  -h, --help                         display help for command
+  -ef, --excluded-files <string...>      excluded files
+  -o, --out <string>                     output folder. Extra markdown fields for output and trace will also be generated
+  -rmo, --remove-out                     remove output folder if it exists
+  -ot, --out-trace <string>              output file for trace
+  -od, --out-data <string>               output file for data (.jsonl/ndjson will be aggregated). JSON schema information and validation will be included if available.
+  -oa, --out-annotations <string>        output file for annotations (.csv will be rendered as csv, .jsonl/ndjson will be aggregated)
+  -ocl, --out-changelog <string>         output file for changelogs
+  -prc, --pull-request-comment [string]  create github comment on a pull request. Use commentid to upsert the comment. (default: "none")
+  -j, --json                             emit full JSON response to output
+  -y, --yaml                             emit full YAML response to output
+  -p, --prompt                           dry run, don't execute LLM and return expanded prompt
+  -fe, --fail-on-errors                  fails on detected annotation error
+  -r, --retry <number>                   number of retries (default: "8")
+  -rd, --retry-delay <number>            minimum delay between retries (default: "15000")
+  -md, --max-delay <number>              maximum delay between retries (default: "180000")
+  -l, --label <string>                   label for the run
+  -t, --temperature <number>             temperature for the run
+  -tp, --top-p <number>                  top-p for the run
+  -m, --model <string>                   model for the run
+  -mt, --max-tokens <number>             maximum tokens for the run
+  -mtc, --max-tool-calls <number>        maximum tool calls for the run
+  -se, --seed <number>                   seed for the run
+  --no-cache                             disable LLM result cache
+  -cn, --cache-name <name>               custom cache file name
+  --cs, --csv-separator <string>         csv separator (default: "\t")
+  -ae, --apply-edits                     apply file edits
+  --vars <namevalue...>                  variables, as name=value, stored in env.vars
+  -h, --help                             display help for command
 ```
 
 ## `batch`

genaisrc/pr-describe.genai.js

@@ -0,0 +1,29 @@
+script({
+    model: "openai:gpt-4",
+    files: [],
+    title: "pr-describe",
+    system: ["system", "system.fs_find_files", "system.fs_read_file"],
+})
+const { stdout: changes } = await host.exec("git", [
+    "diff",
+    "main",
+    "--",
+    ":!**/genaiscript.d.ts",
+])
+
+def("GIT_DIFF", changes, { maxTokens: 20000 })
+
+$`You are an expert software developer and architect.
+
+## Task
+
+- Describe the changes in GIT_DIFF in a way that a software engineer will understand.
+
+## Instructions
+
+- use bullet points to list the changes
+- use emojis to make the description more engaging
+- if needed inline code snippets can be used. The code snippets should be taken
+from the changes in GIT_DIFF.
+
+`

packages/cli/src/cli.ts

@@ -102,8 +102,8 @@ export async function cli() {
         )
         .option("-ocl, --out-changelog <string>", "output file for changelogs")
         .option(
-            "-ghc, --github-comment [string]",
-            "create github comment on a pull request or commit with output. Use commentid to upsert comment.",
+            "-prc, --pull-request-comment [string]",
+            "create github comment on a pull request. Use commentid to upsert the comment.",
             GITHUB_COMMENT_ID_NONE
         )
         .option("-j, --json", "emit full JSON response to output")

packages/cli/src/nodehost.ts

@@ -153,10 +153,14 @@ export class NodeHost implements Host {
         const res = await readFile(name)
         return res ? new Uint8Array(res) : new Uint8Array()
     }
-    async findFiles(path: string): Promise<string[]> {
+    async findFiles(
+        path: string | string[],
+        ignore?: string | string[]
+    ): Promise<string[]> {
         const files = await glob(path, {
             nodir: true,
             windowsPathsNoEscape: true,
+            ignore,
         })
         return files
     }

packages/cli/src/run.ts

@@ -59,7 +59,7 @@ export async function runScript(
         outTrace: string
         outAnnotations: string
         outChangelogs: string
-        githubComment: string
+        pullRequestComment: string
         outData: string
         label: string
         temperature: string
@@ -88,7 +88,7 @@ export async function runScript(
     const outAnnotations = options.outAnnotations
     const failOnErrors = options.failOnErrors
     const outChangelogs = options.outChangelogs
-    const gitHubComment = options.githubComment
+    const pullRequestComment = options.pullRequestComment
     const outData = options.outData
     const label = options.label
     const temperature = normalizeFloat(options.temperature)
@@ -362,16 +362,16 @@ ${Array.from(files)
         }
     }
 
-    if (gitHubComment && res.text) {
+    if (pullRequestComment && res.text) {
         const info = parseGHTokenFromEnv(process.env)
         if (info.repository && info.issue) {
             const ghres = await githubCreateIssueComment(
                 info,
                 prettifyMarkdown(
-                    `${res.text}\n\n> Comment generated by genaiscript ${script.id}.`
+                    `${res.text}\n\n> generated by genaiscript ${script.id}.`
                 ),
-                gitHubComment !== GITHUB_COMMENT_ID_NONE
-                    ? gitHubComment
+                pullRequestComment !== GITHUB_COMMENT_ID_NONE
+                    ? pullRequestComment
                     : undefined
             )
             if (!ghres.created) {

packages/core/src/fs.ts

@@ -2,7 +2,7 @@ import { DOT_ENV_REGEX, HTTPS_REGEX } from "./constants"
 import { NotSupportedError, errorMessage } from "./error"
 import { resolveFileContent } from "./file"
 import { ReadFileOptions, host } from "./host"
-import { logVerbose, utf8Decode, utf8Encode } from "./util"
+import { logVerbose, unique, utf8Decode, utf8Encode } from "./util"
 
 export async function readText(fn: string) {
     const curr = await host.readFile(fn)
@@ -107,26 +107,14 @@ export function createFileSystem(): WorkspaceFileSystem {
 }
 
 export async function expandFiles(files: string[], excludedFiles?: string[]) {
-    const res = new Set<string>()
-    for (const file of files) {
-        if (HTTPS_REGEX.test(file)) res.add(file)
-        // is this a glob?
-        else if (/(\*|\{|\?|\[)/.test(file)) {
-            const fs = await host.findFiles(file)
-            for (const f of fs) res.add(f)
-        } else res.add(file)
-    }
-
-    if (excludedFiles?.length) {
-        for (const arg of excludedFiles) {
-            const ffs = await host.findFiles(arg)
-            for (const f of ffs) {
-                res.delete(f)
-            }
-        }
-    }
-
-    return Array.from(res.values())
+    const urls = files
+        .filter((f) => HTTPS_REGEX.test(f))
+        .filter((f) => !excludedFiles?.includes(f))
+    const others = await host.findFiles(
+        files.filter((f) => !HTTPS_REGEX.test(f)),
+        excludedFiles?.filter((f) => !HTTPS_REGEX.test(f))
+    )
+    return unique([...urls, ...others])
 }
 
 export function filePathOrUrlToWorkspaceFile(f: string) {

packages/core/src/github.ts

@@ -1,7 +1,7 @@
 import { GITHUB_API_VERSION } from "./constants"
 import { createFetch } from "./fetch"
 import { host } from "./host"
-import { normalizeInt } from "./util"
+import { logError, normalizeInt } from "./util"
 
 export interface GithubConnectionInfo {
     token: string
@@ -72,15 +72,19 @@ export async function githubCreateIssueComment(
             id: string
             body: string
         }[]
+
         const comment = comments.find((c) => c.body.includes(tag))
         if (comment) {
-            await fetch(`${url}/${comment.id}`, {
+            const delurl = `${apiUrl}/repos/${repository}/issues/comments/${comment.id}`
+            const resd = await fetch(delurl, {
                 method: "DELETE",
                 headers: {
                     Authorization: `Bearer ${token}`,
                     "X-GitHub-Api-Version": GITHUB_API_VERSION,
                 },
             })
+            if (!resd.ok)
+                logError(`issue comment delete failed ` + resd.statusText)
         }
     }
 

packages/core/src/host.ts

@@ -146,7 +146,7 @@ export interface Host {
     readFile(name: string, options?: ReadFileOptions): Promise<Uint8Array>
     writeFile(name: string, content: Uint8Array): Promise<void>
     deleteFile(name: string): Promise<void>
-    findFiles(glob: string): Promise<string[]>
+    findFiles(pattern: string | string[], ignore?: string | string[]): Promise<string[]>
 
     clearVirtualFiles(): void
     setVirtualFile(name: string, content: string): void

packages/core/src/parser.ts

@@ -20,6 +20,7 @@ type Parser = (
 ) => TextFile
 
 export function stringToPos(str: string): CharPosition {
+    if (!str) return [0, 0]
     return [str.replace(/[^\n]/g, "").length, str.replace(/[^]*\n/, "").length]
 }
 

packages/vscode/src/vshost.ts

@@ -12,6 +12,7 @@ import {
     createBundledParsers,
     AskUserOptions,
     TraceOptions,
+    arrayify,
 } from "genaiscript-core"
 import { Uri } from "vscode"
 import { ExtensionState } from "./state"
@@ -182,9 +183,27 @@ export class VSCodeHost extends EventTarget implements Host {
         delete this.virtualFiles[uri.fsPath]
         await vscode.workspace.fs.delete(uri)
     }
-    async findFiles(path: string): Promise<string[]> {
-        const uris = await vscode.workspace.findFiles(path)
-        return uris.map((u) => vscode.workspace.asRelativePath(u, false))
+    async findFiles(
+        pattern: string | string[],
+        ignore?: string | string[]
+    ): Promise<string[]> {
+        pattern = arrayify(pattern)
+        ignore = arrayify(ignore)
+
+        const uris = new Set<string>()
+        for (const pat of pattern) {
+            const res = await vscode.workspace.findFiles(pat)
+            res.map((u) => vscode.workspace.asRelativePath(u, false)).forEach(
+                (u) => uris.add(u)
+            )
+        }
+        for(const pat of ignore) {
+            const res = await vscode.workspace.findFiles(pat)
+            res.map((u) => vscode.workspace.asRelativePath(u, false)).forEach(
+                (u) => uris.delete(u)
+            )
+        }
+        return Array.from(uris.values())
     }
     async createDirectory(name: string): Promise<void> {
         const uri = this.toProjectFileUri(name)
@@ -205,7 +224,9 @@ export class VSCodeHost extends EventTarget implements Host {
         }
     }
 
-    async getLanguageModelConfiguration(modelId: string): Promise<LanguageModelConfiguration> {
+    async getLanguageModelConfiguration(
+        modelId: string
+    ): Promise<LanguageModelConfiguration> {
         const dotenv = await readFileText(this.projectUri, ".env")
         const env = dotEnvTryParse(dotenv) ?? {}
         const tok = await parseTokenFromEnv(env, modelId)