From e04f6f05e4a1c81d2f04c527138d965a3f066cef Mon Sep 17 00:00:00 2001 From: pbio <10051819+paulbalaji@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:01:02 +0000 Subject: [PATCH 1/3] docs(changeset): Call google storage API directly and remove @google-cloud/storage dependency from the SDK. --- .changeset/many-clouds-bow.md | 5 + .../scripts/check/check-validator-version.ts | 2 + typescript/sdk/package.json | 1 - typescript/sdk/src/gcp/storage.ts | 78 ++++++++----- yarn.lock | 106 +----------------- 5 files changed, 59 insertions(+), 133 deletions(-) create mode 100644 .changeset/many-clouds-bow.md diff --git a/.changeset/many-clouds-bow.md b/.changeset/many-clouds-bow.md new file mode 100644 index 0000000000..b481b543ff --- /dev/null +++ b/.changeset/many-clouds-bow.md @@ -0,0 +1,5 @@ +--- +'@hyperlane-xyz/sdk': minor +--- + +Call google storage API directly and remove @google-cloud/storage dependency from the SDK. diff --git a/typescript/infra/scripts/check/check-validator-version.ts b/typescript/infra/scripts/check/check-validator-version.ts index 72df04621d..f6c1622196 100644 --- a/typescript/infra/scripts/check/check-validator-version.ts +++ b/typescript/infra/scripts/check/check-validator-version.ts @@ -28,6 +28,8 @@ const acceptableValidatorVersions: Record = { 'd834d8147628584acd78a81e344bff76472d707e': 'nov-21-bsquared', // Nov 21 swell/lumiaprism deploy 'b35c105f197267072daa14bb3d83c62410b96fac': 'nov-21-swell', + // Dec 4 deploy + 'a7f3967e047c2c5aabb8cc442e4acad435fa32ab': 'dec-4-batch', // Rolled out only to AW infra before 1.0.0, just 1 commit behind 1.0.0 'a64af8be9a76120d0cfc727bb70660fa07e70cce': 'pre-1.0.0', // 1.0.0 diff --git a/typescript/sdk/package.json b/typescript/sdk/package.json index 47bb713b13..1c383e1839 100644 --- a/typescript/sdk/package.json +++ b/typescript/sdk/package.json @@ -8,7 +8,6 @@ "@chain-registry/types": "^0.50.14", "@cosmjs/cosmwasm-stargate": "^0.32.4", "@cosmjs/stargate": "^0.32.4", - "@google-cloud/storage": "7.14.0", "@hyperlane-xyz/core": "5.8.3", "@hyperlane-xyz/utils": "7.3.0", "@safe-global/api-kit": "1.3.0", diff --git a/typescript/sdk/src/gcp/storage.ts b/typescript/sdk/src/gcp/storage.ts index 96c2a3fd71..42498a81f1 100644 --- a/typescript/sdk/src/gcp/storage.ts +++ b/typescript/sdk/src/gcp/storage.ts @@ -1,5 +1,3 @@ -import { Storage } from '@google-cloud/storage'; - export const GCS_BUCKET_REGEX = /^(?:(?:https?:\/\/)?([^/]+)\.storage\.googleapis\.com\/?|gs:\/\/([^/]+))$/; @@ -18,66 +16,88 @@ export interface StorageConfig { } export class GcpStorageWrapper { - private readonly client: Storage; private readonly bucket: string; private cache: Record> | undefined; + private readonly baseUrl: string; static fromBucketUrl(bucketUrl: string): GcpStorageWrapper { const match = bucketUrl.match(GCS_BUCKET_REGEX); if (!match) throw new Error('Could not parse bucket url'); return new GcpStorageWrapper({ - bucket: match[1], + bucket: match[1] || match[2], // Handle both http and gs:// formats caching: true, }); } constructor(readonly config: StorageConfig) { - this.client = new Storage({ - projectId: config.projectId, - keyFilename: config.keyFilename, - }); this.bucket = config.bucket; + this.baseUrl = `https://storage.googleapis.com/storage/v1/b/${this.bucket}/o`; if (config.caching) { this.cache = {}; } } - formatKey(key: string): string { + private formatKey(key: string): string { return this.config.folder ? `${this.config.folder}/${key}` : key; } - async getObject(key: string): Promise | undefined> { - const Key = this.formatKey(key); - if (this.cache?.[Key]) { - return this.cache![Key]; + private getCachedObject(key: string): StorageReceipt | undefined { + return this.cache?.[key]; + } + + private setCachedObject(key: string, value: StorageReceipt): void { + if (this.cache) { + this.cache[key] = value; } + } - try { - const bucket = this.client.bucket(this.bucket); - const file = bucket.file(Key); - const [exists] = await file.exists(); + private async fetchMetadata(key: string): Promise { + const url = `${this.baseUrl}/${encodeURIComponent(key)}`; + const response = await fetch(url); - if (!exists) { - return undefined; - } + if (response.status === 404) return undefined; + + if (!response.ok) { + throw new Error( + `Failed to fetch object metadata: ${response.statusText}`, + ); + } + + return response.json(); + } + + private async fetchContent(key: string): Promise { + const url = `${this.baseUrl}/${encodeURIComponent(key)}?alt=media`; + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch object content: ${response.statusText}`); + } + + return response.text(); + } - const [metadata] = await file.getMetadata(); - const [contents] = await file.download(); - const body = contents.toString('utf-8'); + async getObject(key: string): Promise | undefined> { + const formattedKey = this.formatKey(key); + const cachedObject = this.getCachedObject(formattedKey); + if (cachedObject) { + return cachedObject; + } + try { + const metadata = await this.fetchMetadata(formattedKey); + if (!metadata) return undefined; + + const body = await this.fetchContent(formattedKey); const result = { data: JSON.parse(body), - // If no updated date is provided, use the Unix epoch start - // 0 = Unix epoch start (1970-01-01T00:00:00.000Z) modified: new Date(metadata.updated ?? 0), }; - if (this.cache) { - this.cache[Key] = result; - } + this.setCachedObject(formattedKey, result); return result; } catch (e: any) { - if (e.code === 404) { + if (e.status === 404) { return undefined; } throw e; diff --git a/yarn.lock b/yarn.lock index 91d745e473..14775828bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7205,30 +7205,6 @@ __metadata: languageName: node linkType: hard -"@google-cloud/paginator@npm:^5.0.0": - version: 5.0.2 - resolution: "@google-cloud/paginator@npm:5.0.2" - dependencies: - arrify: "npm:^2.0.0" - extend: "npm:^3.0.2" - checksum: 10/b64ba2029b77fdcf3c827aea0b6d128122fd1d2f4aa8c1ba70747cba0659d4216a283769fb3bbeb8f726176f5282624637f02c30f118a010e05838411da0cb76 - languageName: node - linkType: hard - -"@google-cloud/projectify@npm:^4.0.0": - version: 4.0.0 - resolution: "@google-cloud/projectify@npm:4.0.0" - checksum: 10/fdccdda0b50855c35541d71c46a6603f3302ff1a00108d946272cb2167435da00e2a2da5963fe489f4f5a4a9eb6320abeb97d3269974a972ae89f5df8451922d - languageName: node - linkType: hard - -"@google-cloud/promisify@npm:^4.0.0": - version: 4.0.0 - resolution: "@google-cloud/promisify@npm:4.0.0" - checksum: 10/c5de81321b3a5c567edcbe0b941fb32644611147f3ba22f20575918c225a979988a99bc2ebda05ac914fa8714b0a54c69be72c3f46c7a64c3b19db7d7fba8d04 - languageName: node - linkType: hard - "@google-cloud/secret-manager@npm:^5.5.0": version: 5.5.0 resolution: "@google-cloud/secret-manager@npm:5.5.0" @@ -7238,29 +7214,6 @@ __metadata: languageName: node linkType: hard -"@google-cloud/storage@npm:7.14.0": - version: 7.14.0 - resolution: "@google-cloud/storage@npm:7.14.0" - dependencies: - "@google-cloud/paginator": "npm:^5.0.0" - "@google-cloud/projectify": "npm:^4.0.0" - "@google-cloud/promisify": "npm:^4.0.0" - abort-controller: "npm:^3.0.0" - async-retry: "npm:^1.3.3" - duplexify: "npm:^4.1.3" - fast-xml-parser: "npm:^4.4.1" - gaxios: "npm:^6.0.2" - google-auth-library: "npm:^9.6.3" - html-entities: "npm:^2.5.2" - mime: "npm:^3.0.0" - p-limit: "npm:^3.0.1" - retry-request: "npm:^7.0.0" - teeny-request: "npm:^9.0.0" - uuid: "npm:^8.0.0" - checksum: 10/0726fde2697da696637fab91ebd756354a58c1331f6a0b9ecc5011de4aae72cd9e1fe3e9564aee15c6a2118e45ed0ae8c3ac9685c6581db6107080f906a949e9 - languageName: node - linkType: hard - "@grpc/grpc-js@npm:~1.10.3": version: 1.10.8 resolution: "@grpc/grpc-js@npm:1.10.8" @@ -7602,7 +7555,6 @@ __metadata: "@cosmjs/cosmwasm-stargate": "npm:^0.32.4" "@cosmjs/stargate": "npm:^0.32.4" "@eslint/js": "npm:^9.15.0" - "@google-cloud/storage": "npm:7.14.0" "@hyperlane-xyz/core": "npm:5.8.3" "@hyperlane-xyz/utils": "npm:7.3.0" "@nomiclabs/hardhat-ethers": "npm:^2.2.3" @@ -17874,13 +17826,6 @@ __metadata: languageName: node linkType: hard -"arrify@npm:^2.0.0": - version: 2.0.1 - resolution: "arrify@npm:2.0.1" - checksum: 10/067c4c1afd182806a82e4c1cb8acee16ab8b5284fbca1ce29408e6e91281c36bb5b612f6ddfbd40a0f7a7e0c75bf2696eb94c027f6e328d6e9c52465c98e4209 - languageName: node - linkType: hard - "as-table@npm:^1.0.36": version: 1.0.55 resolution: "as-table@npm:1.0.55" @@ -21074,7 +21019,7 @@ __metadata: languageName: node linkType: hard -"duplexify@npm:^4.0.0, duplexify@npm:^4.1.2, duplexify@npm:^4.1.3": +"duplexify@npm:^4.0.0, duplexify@npm:^4.1.2": version: 4.1.3 resolution: "duplexify@npm:4.1.3" dependencies: @@ -23202,17 +23147,6 @@ __metadata: languageName: node linkType: hard -"fast-xml-parser@npm:^4.4.1": - version: 4.5.0 - resolution: "fast-xml-parser@npm:4.5.0" - dependencies: - strnum: "npm:^1.0.5" - bin: - fxparser: src/cli/cli.js - checksum: 10/dc9571c10e7b57b5be54bcd2d92f50c446eb42ea5df347d253e94dd14eb99b5300a6d172e840f151e0721933ca2406165a8d9b316a6d777bf0596dc4fe1df756 - languageName: node - linkType: hard - "fastq@npm:^1.6.0": version: 1.13.0 resolution: "fastq@npm:1.13.0" @@ -23955,19 +23889,6 @@ __metadata: languageName: node linkType: hard -"gaxios@npm:^6.0.2": - version: 6.7.1 - resolution: "gaxios@npm:6.7.1" - dependencies: - extend: "npm:^3.0.2" - https-proxy-agent: "npm:^7.0.1" - is-stream: "npm:^2.0.0" - node-fetch: "npm:^2.6.9" - uuid: "npm:^9.0.1" - checksum: 10/c85599162208884eadee91215ebbfa1faa412551df4044626cb561300e15193726e8f23d63b486533e066dadad130f58ed872a23acab455238d8d48b531a0695 - languageName: node - linkType: hard - "gcp-metadata@npm:^6.1.0": version: 6.1.0 resolution: "gcp-metadata@npm:6.1.0" @@ -24475,20 +24396,6 @@ __metadata: languageName: node linkType: hard -"google-auth-library@npm:^9.6.3": - version: 9.15.0 - resolution: "google-auth-library@npm:9.15.0" - dependencies: - base64-js: "npm:^1.3.0" - ecdsa-sig-formatter: "npm:^1.0.11" - gaxios: "npm:^6.1.1" - gcp-metadata: "npm:^6.1.0" - gtoken: "npm:^7.0.0" - jws: "npm:^4.0.0" - checksum: 10/fba2db9732bbf1b3a3a2e2b45131ba8e8aba297377f1c104d0b2ab3386bbc1e02047f20b8a7afca1c6308492da1540104618f1c7b5cd539703552e10399c560e - languageName: node - linkType: hard - "google-gax@npm:^4.0.3": version: 4.3.3 resolution: "google-gax@npm:4.3.3" @@ -25170,13 +25077,6 @@ __metadata: languageName: node linkType: hard -"html-entities@npm:^2.5.2": - version: 2.5.2 - resolution: "html-entities@npm:2.5.2" - checksum: 10/4ec12ebdf2d5ba8192c68e1aef3c1e4a4f36b29246a0a88464fe278a54517d0196d3489af46a3145c7ecacb4fc5fd50497be19eb713b810acab3f0efcf36fdc2 - languageName: node - linkType: hard - "html-escaper@npm:^2.0.0": version: 2.0.2 resolution: "html-escaper@npm:2.0.2" @@ -30113,7 +30013,7 @@ __metadata: languageName: node linkType: hard -"p-limit@npm:^3.0.1, p-limit@npm:^3.0.2, p-limit@npm:^3.1.0": +"p-limit@npm:^3.0.2, p-limit@npm:^3.1.0": version: 3.1.0 resolution: "p-limit@npm:3.1.0" dependencies: @@ -36411,7 +36311,7 @@ __metadata: languageName: node linkType: hard -"uuid@npm:^8.0.0, uuid@npm:^8.3.2": +"uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" bin: From 3f2e88c8247a0339398444f55912714a89079356 Mon Sep 17 00:00:00 2001 From: pbio <10051819+paulbalaji@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:05:52 +0000 Subject: [PATCH 2/3] comments --- typescript/sdk/src/gcp/storage.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/typescript/sdk/src/gcp/storage.ts b/typescript/sdk/src/gcp/storage.ts index 42498a81f1..5f6a610664 100644 --- a/typescript/sdk/src/gcp/storage.ts +++ b/typescript/sdk/src/gcp/storage.ts @@ -24,7 +24,8 @@ export class GcpStorageWrapper { const match = bucketUrl.match(GCS_BUCKET_REGEX); if (!match) throw new Error('Could not parse bucket url'); return new GcpStorageWrapper({ - bucket: match[1] || match[2], // Handle both http and gs:// formats + // Handle both http and gs:// formats + bucket: match[1] || match[2], caching: true, }); } @@ -91,6 +92,8 @@ export class GcpStorageWrapper { const body = await this.fetchContent(formattedKey); const result = { data: JSON.parse(body), + // If no updated date is provided, use the Unix epoch start + // 0 = Unix epoch start (1970-01-01T00:00:00.000Z) modified: new Date(metadata.updated ?? 0), }; From 7ba7051cdd2eff24b06c63fbf0e95b3f308489ff Mon Sep 17 00:00:00 2001 From: pbio <10051819+paulbalaji@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:52:38 +0000 Subject: [PATCH 3/3] CR --- typescript/sdk/src/gcp/storage.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/typescript/sdk/src/gcp/storage.ts b/typescript/sdk/src/gcp/storage.ts index 5f6a610664..6177202314 100644 --- a/typescript/sdk/src/gcp/storage.ts +++ b/typescript/sdk/src/gcp/storage.ts @@ -53,14 +53,15 @@ export class GcpStorageWrapper { } private async fetchMetadata(key: string): Promise { - const url = `${this.baseUrl}/${encodeURIComponent(key)}`; - const response = await fetch(url); + const url = new URL(`${this.baseUrl}/${encodeURIComponent(key)}`); + const response = await fetch(url.toString()); if (response.status === 404) return undefined; if (!response.ok) { + const responseText = await response.text(); throw new Error( - `Failed to fetch object metadata: ${response.statusText}`, + `Failed to fetch object metadata: ${response.statusText}. ${responseText}`, ); } @@ -70,12 +71,15 @@ export class GcpStorageWrapper { private async fetchContent(key: string): Promise { const url = `${this.baseUrl}/${encodeURIComponent(key)}?alt=media`; const response = await fetch(url); + const responseText = await response.text(); if (!response.ok) { - throw new Error(`Failed to fetch object content: ${response.statusText}`); + throw new Error( + `Failed to fetch object content: ${response.statusText}. ${responseText}`, + ); } - return response.text(); + return responseText; } async getObject(key: string): Promise | undefined> {