diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index a02bc812eb..be9ced229a 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -244,3 +244,5 @@ export const TEMPLATE_ARG_DATA_SLICE_SAMPLE = 2000 export const CHAT_REQUEST_PER_MODEL_CONCURRENT_LIMIT = 8 export const PROMISE_QUEUE_CONCURRENCY_DEFAULT = 16 + +export const GITHUB_REST_API_CONCURRENCY_LIMIT = 8 \ No newline at end of file diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/core/src/github.ts b/packages/core/src/github.ts index a956b20f8c..aa172fca54 100644 --- a/packages/core/src/github.ts +++ b/packages/core/src/github.ts @@ -2,14 +2,16 @@ import { Octokit } from "octokit" import { GITHUB_API_VERSION, GITHUB_PULL_REQUEST_REVIEW_COMMENT_LINE_DISTANCE, + GITHUB_REST_API_CONCURRENCY_LIMIT, GITHUB_TOKEN, TOOL_ID, } from "./constants" import { createFetch } from "./fetch" import { runtimeHost } from "./host" import { link, prettifyMarkdown } from "./markdown" -import { assert, logError, logVerbose, normalizeInt } from "./util" +import { arrayify, assert, logError, logVerbose, normalizeInt } from "./util" import { shellRemoveAsciiColors } from "./shell" +import { isGlobMatch } from "./glob" export interface GithubConnectionInfo { token: string @@ -692,4 +694,75 @@ export class GitHubClient implements GitHub { }) return branches.map(({ name }) => name) } + + async listRepositoryLanguages(): Promise> { + const { client, owner, repo } = await this.client() + const { data: languages } = await client.rest.repos.listLanguages({ + owner, + repo, + }) + return languages + } + + async getRepositoryContent( + path: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: string + } + ): Promise { + const { client, owner, repo } = await this.client() + const { ref, type, glob, downloadContent, maxDownloadSize } = + options ?? {} + const { data: contents } = await client.rest.repos.getContent({ + owner, + repo, + path, + ref, + }) + const res = arrayify(contents) + .filter((c) => !type || c.type === type) + .filter((c) => !glob || isGlobMatch(c.path, glob)) + .map((content) => ({ + filename: content.path, + type: content.type, + size: content.size, + content: + content.type === "file" && content.content + ? Buffer.from(content.content, "base64").toString( + "utf-8" + ) + : undefined, + })) + if (downloadContent) { + const q = host.promiseQueue(GITHUB_REST_API_CONCURRENCY_LIMIT) + await q.all( + res + .filter((f) => f.type === "file" && !f.content) + .filter( + (f) => !maxDownloadSize || f.size <= maxDownloadSize + ) + .map((f) => { + const filename = f.filename + return async () => { + const { data: fileContent } = + await client.rest.repos.getContent({ + owner, + repo, + path: filename, + ref, + }) + f.content = Buffer.from( + arrayify(fileContent)[0].content, + "base64" + ).toString("utf8") + } + }) + ) + } + return res + } } diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index e74664fcaf..bfc03b635c 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1349,6 +1349,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1453,6 +1458,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/genaisrc/github.genai.mts b/packages/sample/genaisrc/github.genai.mts index c7fb4c3f5b..fd7d69ac05 100644 --- a/packages/sample/genaisrc/github.genai.mts +++ b/packages/sample/genaisrc/github.genai.mts @@ -2,6 +2,22 @@ script({ model: "openai:gpt-3.5-turbo", tests: {}, }) + +const languages = await github.listRepositoryLanguages() +console.log(languages) + +const files = await github.getRepositoryContent("", { + type: "file", + downloadContent: true, + maxDownloadSize: 2_000, +}) +console.log( + files.map(({ filename, content }) => ({ + filename, + content: content?.slice(0, 50), + })) +) + const issues = await github.listIssues({ per_page: 5 }) console.log(issues.map((i) => i.title)) const issueComments = await github.listIssueComments(issues[0].number) @@ -24,4 +40,4 @@ console.log(runs.map((i) => i.status)) const jobs = await github.listWorkflowJobs(runs[0].id) // redacted job log -console.log(jobs[0].content) \ No newline at end of file +console.log(jobs[0].content) diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD { diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 72d9c5cffc..b294588e9e 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1382,6 +1382,11 @@ interface GitHubPaginationOptions { per_page?: number } +interface GitHubFile extends WorkspaceFile { + type: "file" | "dir" | "submodule" | "symlink" + size: number +} + interface GitHub { /** * Gets connection information for octokit @@ -1486,6 +1491,25 @@ interface GitHub { * Lists branches in a GitHub repository */ listBranches(options?: GitHubPaginationOptions): Promise + + /** + * Lists tags in a GitHub repository + */ + listRepositoryLanguages(): Promise> + + /** + * Lists tags in a GitHub repository + */ + getRepositoryContent( + path?: string, + options?: { + ref?: string + glob?: string + downloadContent?: boolean + maxDownloadSize?: number + type?: (typeof GitHubFile)["type"] + } + ): Promise } interface MD {