From 17f676fffa8ccd548a715af5737ebad2449ba1d2 Mon Sep 17 00:00:00 2001 From: Obafemi Teminife Date: Mon, 4 Nov 2024 16:23:28 +0100 Subject: [PATCH] fix(grants): added proper support for loading grantees from daoip --- package.json | 3 + src/grants/grants.controller.ts | 6 +- src/grants/grants.module.ts | 3 +- src/grants/grants.service.ts | 354 ++++++++++++++++------- src/projects/projects.module.ts | 1 + src/shared/helpers/index.ts | 40 +++ src/shared/interfaces/grant.interface.ts | 36 ++- yarn.lock | 10 + 8 files changed, 336 insertions(+), 117 deletions(-) diff --git a/package.json b/package.json index 1bcbd11d..d3c10312 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "iron-session": "^6.3.1", "joi": "^17.9.2", "js-convert-case": "^4.2.0", + "js-yaml": "^4.1.0", "langchain": "^0.2.17", "lodash": "^4.17.21", "mime": "^3.0.0", @@ -78,6 +79,7 @@ "rxjs": "^7.8.1", "short-unique-id": "^5.2.0", "siwe": "^2.1.4", + "sqids": "^0.3.0", "ts-morph": "^18.0.0", "uuid": "^9.0.1", "viem": "^2.9.15" @@ -93,6 +95,7 @@ "@semantic-release/git": "^10.0.1", "@types/express": "^4.17.17", "@types/jest": "29.5.1", + "@types/js-yaml": "^4.0.9", "@types/lodash": "^4.17.7", "@types/multer": "^1.4.7", "@types/node": "20.1.1", diff --git a/src/grants/grants.controller.ts b/src/grants/grants.controller.ts index 0263a477..95c44ad1 100644 --- a/src/grants/grants.controller.ts +++ b/src/grants/grants.controller.ts @@ -5,7 +5,7 @@ import { ApiUnprocessableEntityResponse, getSchemaPath, } from "@nestjs/swagger"; -import { responseSchemaWrapper } from "src/shared/helpers"; +import { paginate, responseSchemaWrapper } from "src/shared/helpers"; import { Grant, Grantee, @@ -100,7 +100,9 @@ export class GrantsController { this.logger.log( `/grants/${slug}/grantees ${JSON.stringify({ page, limit })}`, ); - return this.grantsService.getGranteesBySlug(slug, page, limit); + return this.grantsService + .getGranteesBySlug(slug) + .then(x => paginate(page, limit, x)); } @Get(":slug/grantees/:granteeSlug") diff --git a/src/grants/grants.module.ts b/src/grants/grants.module.ts index 94de6b5c..f0d9fa7c 100644 --- a/src/grants/grants.module.ts +++ b/src/grants/grants.module.ts @@ -4,9 +4,10 @@ import { GrantsController } from "./grants.controller"; import { GoogleBigQueryModule } from "src/google-bigquery/google-bigquery.module"; import { AuthModule } from "src/auth/auth.module"; import { MailService } from "src/mail/mail.service"; +import { ProjectsModule } from "src/projects/projects.module"; @Module({ - imports: [GoogleBigQueryModule, AuthModule], + imports: [GoogleBigQueryModule, AuthModule, ProjectsModule], controllers: [GrantsController], providers: [GrantsService, MailService], }) diff --git a/src/grants/grants.service.ts b/src/grants/grants.service.ts index 2a698f7f..e38b4f28 100644 --- a/src/grants/grants.service.ts +++ b/src/grants/grants.service.ts @@ -5,6 +5,7 @@ import { Client, createClient } from "./generated"; import { GoogleBigQueryService } from "src/google-bigquery/google-bigquery.service"; import { DaoipFundingData, + DaoipProject, Grantee, GranteeApplicationMetadata, GranteeDetails, @@ -13,6 +14,7 @@ import { GrantProject, KarmaGapGrantProgram, PaginatedData, + RawDaoipProject, RawGrantProjectCodeMetrics, RawGrantProjectContractMetrics, RawGrantProjectOnchainMetrics, @@ -26,12 +28,16 @@ import { sluggify, notStringOrNull, paginate, + uuidfy, + getGoogleLogoUrl, } from "src/shared/helpers"; import { Alchemy, Network } from "alchemy-sdk"; import { Neo4jVectorStore } from "@langchain/community/vectorstores/neo4j_vector"; import { OpenAIEmbeddings } from "@langchain/openai"; import axios from "axios"; import { KARMAGAP_PROGRAM_MAPPINGS } from "src/shared/constants/daoip-karmagap-program-mappings"; +import { ProjectsService } from "src/projects/projects.service"; +import { randomUUID } from "crypto"; @Injectable() export class GrantsService implements OnModuleInit, OnModuleDestroy { @@ -44,6 +50,7 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { @InjectConnection() private readonly neogma: Neogma, private readonly configService: ConfigService, + private readonly projectsService: ProjectsService, private readonly googleBigQueryService: GoogleBigQueryService, ) { this.client = createClient({ @@ -98,9 +105,8 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { } }; - getDaoipFundingDataBySlug = async ( + getDaoipFundingDataByProgramId = async ( programId: string, - granteeProjectSlug: string, ): Promise => { const fundingDataParams = KARMAGAP_PROGRAM_MAPPINGS[programId]; @@ -110,12 +116,70 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { x.from_funder_name === fundingDataParams?.from_funder_name && x.grant_pool_name === fundingDataParams?.grant_pool_name, ); + return daiopFundingData; + }; + + getDaoipFundingDataBySlug = async ( + programId: string, + granteeProjectSlug: string, + ): Promise => { + const daiopFundingData = await this.getDaoipFundingDataByProgramId( + programId, + ); return daiopFundingData.filter( x => x.to_project_name === granteeProjectSlug, ); }; + getDaoipProjectBySlug = async ( + slug: string, + ): Promise => { + try { + const result = await axios.get( + `https://raw.githubusercontent.com/opensource-observer/oss-directory/refs/heads/main/data/projects/${slug.charAt( + 0, + )}/${slug}.yaml`, + ); + if (result.status === 200) { + const yaml = await import("js-yaml"); + const yamlData = yaml.load(result.data) as RawDaoipProject; + const project = { + id: randomUUID(), + name: yamlData.name, + slug, + description: yamlData.description ?? null, + website: yamlData.websites?.[0]?.url ?? null, + twitter: yamlData.social?.twitter?.[0]?.url ?? null, + github: yamlData.github?.[0]?.url ?? null, + }; + + if (project.website) { + await this.projectsService.addProjectByUrl({ + url: project.website, + name: project.name, + }); + return project; + } else { + return project; + } + } else { + this.logger.warn(`Could not find project ${slug} on daoip`); + return null; + } + } catch (err) { + Sentry.withScope(scope => { + scope.setTags({ + action: "db-call", + source: "grants.service", + }); + Sentry.captureException(err); + }); + this.logger.error(`GrantsService::getDaoipProjectBySlug ${err.message}`); + return null; + } + }; + query = async ( query: string, page = 1, @@ -351,11 +415,7 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { } }; - getGranteesBySlug = async ( - slug: string, - page: number, - limit: number, - ): Promise> => { + getGranteesBySlug = async (slug: string): Promise => { try { const result = await this.neogma.queryRunner.run( ` @@ -405,7 +465,7 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { }, }); - const grantees = + const gitcoinGrantees = program.programId === "451" ? result.rounds.find( x => @@ -416,44 +476,66 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { x => (x.roundMetadata as GrantMetadata)?.name === program.name, )?.applications ?? []; - return paginate( - page, - limit, - await Promise.all( - grantees.map(async grantee => { - const apiKey = this.configService.get("ALCHEMY_API_KEY"); + const gitcoinGranteesFinal = await Promise.all( + gitcoinGrantees.map(async grantee => { + const apiKey = this.configService.get("ALCHEMY_API_KEY"); - const alchemy = new Alchemy({ - apiKey, - network: Network.ARB_MAINNET, - }); + const alchemy = new Alchemy({ + apiKey, + network: Network.ARB_MAINNET, + }); - const transaction = grantee.distributionTransaction - ? await alchemy.core.getTransaction( - grantee.distributionTransaction, - ) - : { - timestamp: 0, - }; + const transaction = grantee.distributionTransaction + ? await alchemy.core.getTransaction( + grantee.distributionTransaction, + ) + : { + timestamp: 0, + }; - const logoIpfs = notStringOrNull( - (grantee.metadata as GranteeApplicationMetadata)?.application - .project.logoImg, - ); + const logoIpfs = notStringOrNull( + (grantee.metadata as GranteeApplicationMetadata)?.application + .project.logoImg, + ); - return new Grantee({ - id: grantee.id, - name: grantee.project.name, - slug: sluggify(grantee.project.name), - logoUrl: logoIpfs ? `https://${logoIpfs}.ipfs.dweb.link` : null, - lastFundingDate: nonZeroOrNull(transaction.timestamp), - lastFundingAmount: grantee.totalAmountDonatedInUsd, - }); - }), - ), + return new Grantee({ + id: grantee.id, + name: grantee.project.name, + slug: sluggify(grantee.project.name), + logoUrl: logoIpfs ? `https://${logoIpfs}.ipfs.dweb.link` : null, + lastFundingDate: nonZeroOrNull(transaction.timestamp), + lastFundingAmount: `${grantee.totalAmountDonatedInUsd} USD`, + }); + }), + ); + + const daoipGrantees = await this.getDaoipFundingDataByProgramId( + program.programId, ); + + const daoipGranteesFinal = await Promise.all( + daoipGrantees.map(async x => { + const project = await this.getDaoipProjectBySlug( + x.to_project_name ?? sluggify(x.metadata.application_name), + ); + return new Grantee({ + id: randomUUID(), + name: x.metadata.application_name, + slug: x.to_project_name ?? sluggify(x.metadata.application_name), + logoUrl: project?.website + ? getGoogleLogoUrl(project.website) + : null, + lastFundingDate: nonZeroOrNull( + new Date(x.funding_date).getTime(), + ), + lastFundingAmount: `${x.amount} ${x.metadata.token_unit}`, + }); + }), + ); + + return [...gitcoinGranteesFinal, ...daoipGranteesFinal]; } else { - return paginate(page, limit, []); + return []; } } catch (err) { Sentry.withScope(scope => { @@ -464,7 +546,7 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { Sentry.captureException(err); }); this.logger.error(`GrantsService::getGranteesByProgramId ${err.message}`); - return paginate(page, limit, []); + return []; } }; @@ -523,7 +605,7 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { const program = programs[0]; - const project = ( + const gitcoinDetails = ( program.programId === "451" ? result.rounds.find( x => @@ -537,6 +619,11 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { return sluggify(x.project.name) === granteeSlug; }); + const daoipDetails = await this.getDaoipFundingDataBySlug( + program.programId, + granteeSlug, + ); + const [codeMetrics, onChainMetrics, contractMetrics] = await Promise.all([ this.googleBigQueryService.getGrantProjectsCodeMetrics([ @@ -562,11 +649,6 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { m => m.project_name === granteeSlug, ) ?? {}) as RawGrantProjectContractMetrics; - const projectDaoipFundingData = await this.getDaoipFundingDataBySlug( - program.programId, - granteeSlug, - ); - const projectMetricsConverter = ( codeMetrics: RawGrantProjectCodeMetrics, onChainMetrics: RawGrantProjectOnchainMetrics, @@ -871,11 +953,11 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { })), } : null, - projectDaoipFundingData.length > 0 + daoipDetails.length > 0 ? { label: "DAOIP Funding Data", tab: "daoip-funding-data", - stats: projectDaoipFundingData.flatMap(x => [ + stats: daoipDetails.flatMap(x => [ { label: "Funding Amount", value: `${x.amount} ${x.metadata.token_unit}`, @@ -897,78 +979,126 @@ export class GrantsService implements OnModuleInit, OnModuleDestroy { ].filter(Boolean); }; - const grantees = ( - program.programId === "451" - ? result.rounds.find( - x => - (x.roundMetadata as GrantMetadata)?.name === - "GG21: Thriving Arbitrum Summer", - )?.applications ?? [] - : result.rounds.find( - x => (x.roundMetadata as GrantMetadata)?.name === program.name, - )?.applications ?? [] - ).map(async grantee => { - const apiKey = this.configService.get("ALCHEMY_API_KEY"); + let granteeDetails: GranteeDetails | null = null; - const alchemy = new Alchemy({ - apiKey, - network: Network.ARB_MAINNET, - }); + if (gitcoinDetails) { + const gitcoinGrantees = ( + program.programId === "451" + ? result.rounds.find( + x => + (x.roundMetadata as GrantMetadata)?.name === + "GG21: Thriving Arbitrum Summer", + )?.applications ?? [] + : result.rounds.find( + x => + (x.roundMetadata as GrantMetadata)?.name === program.name, + )?.applications ?? [] + ).map(async grantee => { + const apiKey = this.configService.get("ALCHEMY_API_KEY"); - const transaction = grantee.distributionTransaction - ? await alchemy.core.getTransaction(grantee.distributionTransaction) - : { - timestamp: 0, - }; + const alchemy = new Alchemy({ + apiKey, + network: Network.ARB_MAINNET, + }); - const logoIpfs = notStringOrNull( - (grantee?.metadata as GranteeApplicationMetadata)?.application - .project.logoImg, - ); - return new GranteeDetails({ - id: grantee.id, - tags: grantee.tags, - status: grantee.status, - name: grantee.project.name, - slug: sluggify(grantee.project.name), - description: (grantee?.metadata as GranteeApplicationMetadata) - ?.application?.project?.description, - website: notStringOrNull( + const transaction = grantee.distributionTransaction + ? await alchemy.core.getTransaction( + grantee.distributionTransaction, + ) + : { + timestamp: 0, + }; + + const logoIpfs = notStringOrNull( (grantee?.metadata as GranteeApplicationMetadata)?.application - ?.project?.website, - ), - logoUrl: logoIpfs ? `https://${logoIpfs}.ipfs.dweb.link` : null, - lastFundingDate: nonZeroOrNull(transaction.timestamp), - lastFundingAmount: grantee.totalAmountDonatedInUsd, - projects: [ - { - id: (project?.metadata as GranteeApplicationMetadata) - ?.application?.project?.id, - name: `GG21: Thriving Arbitrum Summer - ${ - project?.project?.name ?? granteeSlug - }`, - tags: project?.project?.tags ?? [], - tabs: projectCodeMetrics - ? projectMetricsConverter( - projectCodeMetrics, - projectOnchainMetrics, - projectContractMetrics, - ) - : [], - }, - ], + .project.logoImg, + ); + return new GranteeDetails({ + id: grantee.id, + tags: grantee.tags, + status: grantee.status, + name: grantee.project.name, + slug: sluggify(grantee.project.name), + description: (grantee?.metadata as GranteeApplicationMetadata) + ?.application?.project?.description, + website: notStringOrNull( + (grantee?.metadata as GranteeApplicationMetadata)?.application + ?.project?.website, + ), + logoUrl: logoIpfs ? `https://${logoIpfs}.ipfs.dweb.link` : null, + lastFundingDate: nonZeroOrNull(transaction.timestamp), + lastFundingAmount: `${grantee.totalAmountDonatedInUsd} USD`, + projects: [ + gitcoinDetails + ? { + id: ( + gitcoinDetails?.metadata as GranteeApplicationMetadata + )?.application?.project?.id, + name: `GG21: Thriving Arbitrum Summer - ${ + gitcoinDetails?.project?.name ?? granteeSlug + }`, + tags: gitcoinDetails?.project?.tags ?? [], + tabs: projectCodeMetrics + ? projectMetricsConverter( + projectCodeMetrics, + projectOnchainMetrics, + projectContractMetrics, + ) + : [], + } + : null, + ].filter(Boolean), + }); }); - }); - - const grantee = (await Promise.all(grantees)).find( - x => x.slug === granteeSlug, - ); + granteeDetails = (await Promise.all(gitcoinGrantees)).find( + x => x.slug === granteeSlug, + ); + } else { + const project = await this.getDaoipProjectBySlug(granteeSlug); + if (project) { + const details = daoipDetails.find( + x => + x.to_project_name === granteeSlug || + sluggify(x.metadata.application_name) === granteeSlug, + ); + granteeDetails = new GranteeDetails({ + id: uuidfy(granteeSlug), + tags: [], + status: "APPROVED", + name: project?.name, + slug: granteeSlug, + description: project?.description ?? null, + website: project?.website ?? null, + logoUrl: project?.website + ? getGoogleLogoUrl(project.website) + : null, + lastFundingDate: details?.funding_date + ? nonZeroOrNull(new Date(details?.funding_date).getTime()) + : null, + lastFundingAmount: `${details?.amount} ${details?.metadata.token_unit}`, + projects: [ + { + id: uuidfy(granteeSlug), + name: project?.name ?? granteeSlug, + tags: [], + tabs: projectCodeMetrics + ? projectMetricsConverter( + projectCodeMetrics, + projectOnchainMetrics, + projectContractMetrics, + ) + : [], + }, + ], + }); + } + } - if (grantee) { + if (granteeDetails) { return { success: true, message: "Grantee retrieved successfully", - data: grantee, + data: granteeDetails, }; } else { return { diff --git a/src/projects/projects.module.ts b/src/projects/projects.module.ts index c317ea2c..5ae08fb3 100644 --- a/src/projects/projects.module.ts +++ b/src/projects/projects.module.ts @@ -15,5 +15,6 @@ import { AuthModule } from "src/auth/auth.module"; OrganizationsService, ModelService, ], + exports: [ProjectsService], }) export class ProjectsModule {} diff --git a/src/shared/helpers/index.ts b/src/shared/helpers/index.ts index b5f31880..5c765a6e 100644 --- a/src/shared/helpers/index.ts +++ b/src/shared/helpers/index.ts @@ -34,6 +34,7 @@ import { Response } from "../interfaces/response.interface"; import { PUBLIC_API_SCHEMAS } from "../presets/public-api-schemas"; import { CustomLogger } from "../utils/custom-logger"; import { emojiRegex } from "./emoji-regex"; +import Sqids from "sqids"; /* optionalMinMaxFilter is a function that conditionally applies a filter to a cypher query if min or max numeric values are set. @@ -541,6 +542,13 @@ export const randomToken = (): string => { return generate; }; +export const uuidfy = (from: string, minLength = 64): string => { + const sqids = new Sqids({ + minLength, + }); + return sqids.encode(from.split("-").map(x => x.charCodeAt(0))); +}; + export const obfuscate = (value: string | null): string | null => { if (value) { const lastFour = value.slice(-4); @@ -601,3 +609,35 @@ export const toAbsoluteURL = (url: string, baseUrl?: string): string => { } } }; + +export const getWebsiteText = ( + website: string | null, +): { link: string; hostname: string } => { + if (!website) return { link: "", hostname: "" }; + + const isUrl = website.startsWith("http"); + const url = new URL(isUrl ? website : `https://${website}`); + + return { + link: url.toString(), + hostname: url.hostname, + }; +}; + +const URL_PREFIX = "https://www.google.com/s2/favicons?domain="; +const URL_SUFFIX = "&sz=64"; + +export const getGoogleLogoUrl = (url: string | null): string => + `${URL_PREFIX}${getWebsiteText(url).hostname}${URL_SUFFIX}`; + +export const getLogoUrlHttpsAlternative = ( + googleString: string, + frontendUrl: string, +): string => { + const url = new URL( + `${googleString.startsWith("http") ? "" : frontendUrl}${googleString}`, + ); + const domain = url.searchParams.get("domain"); + + return `${URL_PREFIX}https://${domain}${URL_SUFFIX}`; +}; diff --git a/src/shared/interfaces/grant.interface.ts b/src/shared/interfaces/grant.interface.ts index 8dfc0496..819c213c 100644 --- a/src/shared/interfaces/grant.interface.ts +++ b/src/shared/interfaces/grant.interface.ts @@ -347,7 +347,7 @@ export class Grantee { slug: t.string, logoUrl: t.union([t.string, t.null]), lastFundingDate: t.union([t.number, t.null]), - lastFundingAmount: t.number, + lastFundingAmount: t.string, }); @ApiProperty() @@ -366,7 +366,7 @@ export class Grantee { lastFundingDate: number | null; @ApiProperty() - lastFundingAmount: number; + lastFundingAmount: string; constructor(raw: Grantee) { const { id, name, slug, logoUrl, lastFundingDate, lastFundingAmount } = raw; @@ -669,3 +669,35 @@ export interface DaoipFundingDataMetadata { token_amount: number; token_unit: string; } + +export interface DaoipUrl { + url: string; +} + +export interface DaoipSocial { + twitter: DaoipUrl[] | null; + farcaster: DaoipUrl[] | null; + telegram: DaoipUrl[] | null; + medium: DaoipUrl[] | null; + mirror: DaoipUrl[] | null; +} + +export interface RawDaoipProject { + version: string; + name: string; + display_name: string; + description: string | null; + websites: DaoipUrl[] | null; + social: DaoipSocial | null; + github: DaoipUrl[] | null; +} + +export interface DaoipProject { + id: string; + name: string; + slug: string; + description: string | null; + website: string | null; + twitter: string | null; + github: string | null; +} diff --git a/yarn.lock b/yarn.lock index 520446ad..29236f82 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2417,6 +2417,11 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/js-yaml@^4.0.9": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.9.tgz#cd82382c4f902fed9691a2ed79ec68c5898af4c2" + integrity sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg== + "@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" @@ -10126,6 +10131,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== +sqids@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/sqids/-/sqids-0.3.0.tgz#969eabe54a362682e44cb73fce2bfea84e504dec" + integrity sha512-lOQK1ucVg+W6n3FhRwwSeUijxe93b51Bfz5PMRMihVf1iVkl82ePQG7V5vwrhzB11v0NtsR25PSZRGiSomJaJw== + ssri@^10.0.0, ssri@^10.0.1, ssri@^10.0.4: version "10.0.5" resolved "https://registry.yarnpkg.com/ssri/-/ssri-10.0.5.tgz#e49efcd6e36385196cb515d3a2ad6c3f0265ef8c"