Add GitHub API concurrency limit and functions to list repository lan…

…guages and content

packages/core/src/constants.ts

@@ -244,3 +244,5 @@ export const TEMPLATE_ARG_DATA_SLICE_SAMPLE = 2000
 
 export const CHAT_REQUEST_PER_MODEL_CONCURRENT_LIMIT = 8
 export const PROMISE_QUEUE_CONCURRENCY_DEFAULT = 16
+
+export const GITHUB_REST_API_CONCURRENCY_LIMIT = 8

packages/core/src/github.ts

@@ -2,14 +2,16 @@ import { Octokit } from "octokit"
 import {
     GITHUB_API_VERSION,
     GITHUB_PULL_REQUEST_REVIEW_COMMENT_LINE_DISTANCE,
+    GITHUB_REST_API_CONCURRENCY_LIMIT,
     GITHUB_TOKEN,
     TOOL_ID,
 } from "./constants"
 import { createFetch } from "./fetch"
 import { runtimeHost } from "./host"
 import { link, prettifyMarkdown } from "./markdown"
-import { assert, logError, logVerbose, normalizeInt } from "./util"
+import { arrayify, assert, logError, logVerbose, normalizeInt } from "./util"
 import { shellRemoveAsciiColors } from "./shell"
+import { isGlobMatch } from "./glob"
 
 export interface GithubConnectionInfo {
     token: string
@@ -692,4 +694,75 @@ export class GitHubClient implements GitHub {
         })
         return branches.map(({ name }) => name)
     }
+
+    async listRepositoryLanguages(): Promise<Record<string, number>> {
+        const { client, owner, repo } = await this.client()
+        const { data: languages } = await client.rest.repos.listLanguages({
+            owner,
+            repo,
+        })
+        return languages
+    }
+
+    async getRepositoryContent(
+        path: string,
+        options?: {
+            ref?: string
+            glob?: string
+            downloadContent?: boolean
+            maxDownloadSize?: number
+            type?: string
+        }
+    ): Promise<GitHubFile[]> {
+        const { client, owner, repo } = await this.client()
+        const { ref, type, glob, downloadContent, maxDownloadSize } =
+            options ?? {}
+        const { data: contents } = await client.rest.repos.getContent({
+            owner,
+            repo,
+            path,
+            ref,
+        })
+        const res = arrayify(contents)
+            .filter((c) => !type || c.type === type)
+            .filter((c) => !glob || isGlobMatch(c.path, glob))
+            .map((content) => ({
+                filename: content.path,
+                type: content.type,
+                size: content.size,
+                content:
+                    content.type === "file" && content.content
+                        ? Buffer.from(content.content, "base64").toString(
+                              "utf-8"
+                          )
+                        : undefined,
+            }))
+        if (downloadContent) {
+            const q = host.promiseQueue(GITHUB_REST_API_CONCURRENCY_LIMIT)
+            await q.all(
+                res
+                    .filter((f) => f.type === "file" && !f.content)
+                    .filter(
+                        (f) => !maxDownloadSize || f.size <= maxDownloadSize
+                    )
+                    .map((f) => {
+                        const filename = f.filename
+                        return async () => {
+                            const { data: fileContent } =
+                                await client.rest.repos.getContent({
+                                    owner,
+                                    repo,
+                                    path: filename,
+                                    ref,
+                                })
+                            f.content = Buffer.from(
+                                arrayify(fileContent)[0].content,
+                                "base64"
+                            ).toString("utf8")
+                        }
+                    })
+            )
+        }
+        return res
+    }
 }

packages/core/src/types/prompt_template.d.ts

@@ -1349,6 +1349,11 @@ interface GitHubPaginationOptions {
     per_page?: number
 }
 
+interface GitHubFile extends WorkspaceFile {
+    type: "file" | "dir" | "submodule" | "symlink"
+    size: number
+}
+
 interface GitHub {
     /**
      * Gets connection information for octokit
@@ -1453,6 +1458,25 @@ interface GitHub {
      * Lists branches in a GitHub repository
      */
     listBranches(options?: GitHubPaginationOptions): Promise<string[]>
+
+    /**
+     * Lists tags in a GitHub repository
+     */
+    listRepositoryLanguages(): Promise<Record<string, number>>
+
+    /**
+     * Lists tags in a GitHub repository
+     */
+    getRepositoryContent(
+        path?: string,
+        options?: {
+            ref?: string
+            glob?: string
+            downloadContent?: boolean
+            maxDownloadSize?: number
+            type?: (typeof GitHubFile)["type"]
+        }
+    ): Promise<GitHubFile[]>
 }
 
 interface MD {

packages/sample/genaisrc/github.genai.mts

@@ -2,6 +2,22 @@ script({
     model: "openai:gpt-3.5-turbo",
     tests: {},
 })
+
+const languages = await github.listRepositoryLanguages()
+console.log(languages)
+
+const files = await github.getRepositoryContent("", {
+    type: "file",
+    downloadContent: true,
+    maxDownloadSize: 2_000,
+})
+console.log(
+    files.map(({ filename, content }) => ({
+        filename,
+        content: content?.slice(0, 50),
+    }))
+)
+
 const issues = await github.listIssues({ per_page: 5 })
 console.log(issues.map((i) => i.title))
 const issueComments = await github.listIssueComments(issues[0].number)
@@ -24,4 +40,4 @@ console.log(runs.map((i) => i.status))
 
 const jobs = await github.listWorkflowJobs(runs[0].id)
 // redacted job log
-console.log(jobs[0].content)
+console.log(jobs[0].content)