From 673f8f3f213d720390efc9d1b76f2f3a9972d65c Mon Sep 17 00:00:00 2001 From: 0x73746F66 Date: Sat, 15 Jun 2024 23:21:31 +1000 Subject: [PATCH] feat: r2 as gh data cache --- functions/github/repos/cached.js | 237 +++++++++++++++++++++---------- src/pages/GitHub.vue | 22 ++- 2 files changed, 169 insertions(+), 90 deletions(-) diff --git a/functions/github/repos/cached.js b/functions/github/repos/cached.js index 58bc7d0..b322d12 100644 --- a/functions/github/repos/cached.js +++ b/functions/github/repos/cached.js @@ -1,4 +1,4 @@ -import { CloudFlare } from "../../../src/utils" +import { CloudFlare } from "../../src/utils" const cf = new CloudFlare() @@ -28,79 +28,164 @@ export async function onRequestGet(context) { if (session?.expiry <= +new Date()) { return Response.json({ 'err': 'Expired' }) } - // try { - // const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) - // const githubApps = await cf.d1all(env.d1db, "SELECT * FROM github_apps WHERE memberEmail = ?", session.memberEmail) - // let installs = [] - // for (const app of githubApps) { - // if (!app.accessToken) { - // console.log(`github_apps kid=${token} installationId=${app.installationId}`) - // throw new Error('github_apps invalid') - // } - // const gh = new GitHub(app.accessToken) - // const prefixRepos = `/github/${app.installationId}/repos/` - // let repoCache = await cf.r2list(env.r2icache, prefixRepos) - - // const repos = [] - // for (const repo of await gh.getRepos()) { - // const pathSuffix = `${repo.full_name}/${repo.id}.json` - // const repoMetadata = repoCache.filter(r => r.key.endsWith(pathSuffix)) - // if (repoMetadata.length === 0) { - // await cf.PPPPUUUUUUTTTTTTT(env.r2icache, `${prefixRepos}${pathSuffix}`, repo) - // } - - // const data = { - // ghid: repo.id, - // fullName: repo.full_name, - // createdAt: repo.created_at, - // visibility: repo.visibility, - // archived: repo.archived, - // defaultBranch: repo.default_branch, - // pushedAt: repo.pushed_at, - // avatarUrl: repo.owner.avatar_url, - // license: repo.license, - // } - // const prefixBranches = `/github/${app.installationId}/branches/${repo.full_name}/` - - // for (const branch of await gh.getBranches(repo)) { - // const branchData = Object.assign({}, data) - // const r2Branch = await cf.r2get(env.r2icache, `${prefixBranches}`) - // if (r2Branch) { - // repos.push(await r2Branch.json()) - // continue - // } - // branchData.branch = branch.name - // branchData.latestCommitSHA = branch.commit.sha - - // // if (repo.default_branch === branch.name) { - // // const latestCommit = await gh.getCommit(repo, branch) - - // // branchData.latestCommitMessage = latestCommit.commit.message - // // branchData.latestCommitVerification = latestCommit.commit.verification - // // branchData.latestCommitter = latestCommit.commit.committer - // // branchData.latestStats = latestCommit.stats - // // branchData.latestFilesChanged = latestCommit.files.length - - // // const fileDetails = await gh.getFileContents(repo, branch) - - // // branchData.dotfileExists = fileDetails.exists - // // branchData.dotfileContents = fileDetails.content - // // } - // await cf.PPPPUUUUUUTTTTTTT(env.r2icache, `${prefixRepos}${pathSuffix}`, repo) - // repos.push(branchData) - // } - // } - // installs = installs.concat({ - // repos, - // installationId: app.installationId, - // created: app.created, - // }) - // } - - // return Response.json(installs) - // } catch (e) { - // console.error(e) - - // return Response.json(e) - // } + try { + const githubApps = await cf.d1all(env.d1db, "SELECT * FROM github_apps WHERE memberEmail = ?", session.memberEmail) + let installs = [] + + for (const app of githubApps) { + const prefixRepos = `github/${app.installationId}/repos/` + const repoCache = await cf.r2list(env.r2icache, prefixRepos) + const repos = [] + for (const objectKeyRepo of repoCache.map(r => r.key)) { + const repoMetadata = await cf.r2get(env.r2icache, objectKeyRepo) + if (repoMetadata) { + const repo = await repoMetadata.json() + const data = { + ghid: repo.id, + fullName: repo.full_name, + branch: repo.default_branch, + createdAt: repo.created_at, + visibility: repo.visibility, + archived: repo.archived, + defaultBranch: repo.default_branch, + pushedAt: repo.pushed_at, + avatarUrl: repo.owner.avatar_url, + license: repo.license, + } + const branchCache = await cf.r2list(env.r2icache, `/github/${app.installationId}/branches/${repo.full_name}/`) + for (const objectKeyBranch of branchCache.map(b => b.key)) { + const branchMetadata = await cf.r2get(env.r2icache, objectKeyBranch) + if (branchMetadata) { + const branch = await branchMetadata.json() + data.latestCommitSHA = branch?.commit?.sha + data.branch = branch?.name + // data.latestCommitMessage = branch?.commit?.message + // data.latestCommitVerification = branch?.commit?.verification + // data.latestCommitter = branch?.commit?.committer + // data.latestStats = branch?.stats + // data.latestFilesChanged = branch?.files?.length + // data.dotfileExists = branch?.exists + // data.dotfileContents = branch?.content + repos.push(data) + } + } + } + } + installs = installs.concat({ + repos, + installationId: app.installationId, + created: app.created, + }) + } + + return Response.json(installs) + + } catch (e) { + console.error(e) + + return Response.json(e) + } } + +// async function _onRequestGet(context) { +// const { +// request, // same as existing Worker API +// env, // same as existing Worker API +// params, // if filename includes [id] or [[path]] +// waitUntil, // same as ctx.waitUntil in existing Worker API +// next, // used for middleware or to fetch assets +// data, // arbitrary space for passing data between middlewares +// } = context + +// const token = request.headers.get('x-trivialsec') +// if (!token) { +// return Response.json({ 'err': 'Forbidden' }) +// } + +// const session = await env.d1db.prepare("SELECT memberEmail, expiry FROM sessions WHERE kid = ?") +// .bind(token) +// .first() + +// console.log('session expiry', session?.expiry) +// if (!session) { +// return Response.json({ 'err': 'Revoked' }) +// } +// if (session?.expiry <= +new Date()) { +// return Response.json({ 'err': 'Expired' }) +// } +// try { +// const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) +// const githubApps = await cf.d1all(env.d1db, "SELECT * FROM github_apps WHERE memberEmail = ?", session.memberEmail) +// let installs = [] +// for (const app of githubApps) { +// if (!app.accessToken) { +// console.log(`github_apps kid=${token} installationId=${app.installationId}`) +// throw new Error('github_apps invalid') +// } +// const gh = new GitHub(app.accessToken) +// const prefixRepos = `/github/${app.installationId}/repos/` +// let repoCache = await cf.r2list(env.r2icache, prefixRepos) + +// const repos = [] +// for (const repo of await gh.getRepos()) { +// const pathSuffix = `${repo.full_name}/${repo.id}.json` +// const repoMetadata = repoCache.filter(r => r.key.endsWith(pathSuffix)) +// if (repoMetadata.length === 0) { +// await cf.PPPPUUUUUUTTTTTTT(env.r2icache, `${prefixRepos}${pathSuffix}`, repo) +// } + +// const data = { +// ghid: repo.id, +// fullName: repo.full_name, +// createdAt: repo.created_at, +// visibility: repo.visibility, +// archived: repo.archived, +// defaultBranch: repo.default_branch, +// pushedAt: repo.pushed_at, +// avatarUrl: repo.owner.avatar_url, +// license: repo.license, +// } +// const prefixBranches = `/github/${app.installationId}/branches/${repo.full_name}/` + +// for (const branch of await gh.getBranches(repo)) { +// const branchData = Object.assign({}, data) +// const r2Branch = await cf.r2get(env.r2icache, `${prefixBranches}`) +// if (r2Branch) { +// repos.push(await r2Branch.json()) +// continue +// } +// branchData.branch = branch.name +// branchData.latestCommitSHA = branch.commit.sha + +// // if (repo.default_branch === branch.name) { +// // const latestCommit = await gh.getCommit(repo, branch) + +// // branchData.latestCommitMessage = latestCommit.commit.message +// // branchData.latestCommitVerification = latestCommit.commit.verification +// // branchData.latestCommitter = latestCommit.commit.committer +// // branchData.latestStats = latestCommit.stats +// // branchData.latestFilesChanged = latestCommit.files.length + +// // const fileDetails = await gh.getFileContents(repo, branch) + +// // branchData.dotfileExists = fileDetails.exists +// // branchData.dotfileContents = fileDetails.content +// // } +// await cf.PPPPUUUUUUTTTTTTT(env.r2icache, `${prefixRepos}${pathSuffix}`, repo) +// repos.push(branchData) +// } +// } +// installs = installs.concat({ +// repos, +// installationId: app.installationId, +// created: app.created, +// }) +// } + +// return Response.json(installs) +// } catch (e) { +// console.error(e) + +// return Response.json(e) +// } +// } diff --git a/src/pages/GitHub.vue b/src/pages/GitHub.vue index 71e9f41..c33f66e 100644 --- a/src/pages/GitHub.vue +++ b/src/pages/GitHub.vue @@ -38,7 +38,7 @@ class GitHub { if (this.urlQuery?.setup_action === 'install' && this.urlQuery?.code && this.urlQuery?.installation_id) { this.install(this.urlQuery.code, this.urlQuery.installation_id) } else if (!state.cached) { - this.refreshRepos() + this.refreshRepos(true) } } async install(code, installation_id) { @@ -49,11 +49,15 @@ class GitHub { return setTimeout(state.success = "GitHub App installed successfully.", 1000) } - async refreshRepos() { + async refreshRepos(cached = false) { clearAlerts() state.loading = true try { - const { data } = await axios.get('/github/repos') + let uriPath = '/github/repos' + if (cached === true) { + uriPath += '/cached' + } + const { data } = await axios.get(uriPath) state.loading = false if (typeof data === "string" && !isJSON(data)) { @@ -76,7 +80,6 @@ class GitHub { } else { state.apps = data state.success = "Refreshed GitHub repositories" - localStorage.setItem('/github/installs', JSON.stringify(data)) state.cached = true } @@ -141,15 +144,6 @@ function clearAlerts() { state.success = '' } -function loadCached() { - clearAlerts() - const stored = localStorage.getItem('/github/installs') - state.apps = isJSON(stored) ? JSON.parse(stored) : [] - state.installs = state.apps.length > 0 - state.cached = state.apps.map(i => i.repos.length).reduce((a, b) => a + b, 0) > 0 -} -loadCached() - const gh = reactive(new GitHub()) @@ -209,7 +203,7 @@ const gh = reactive(new GitHub()) prepend-icon="line-md:downloading-loop" variant="text" :color="global.name.value === 'dark' ? '#fff' : '#272727'" - @click="loadCached" + @click="gh.refreshRepos(true)" />