diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f2514582a0..770b1ee99b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,17 +1,16 @@ name: Setup -description: Sets up a Node.js environment, installs the dependencies using Yarn, +description: Sets up a Node.js environment, installs the dependencies using Yarn, and caches the installed dependencies for faster future builds. runs: using: composite steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 18 registry-url: https://registry.npmjs.org cache: yarn - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: install-cache with: path: node_modules/ @@ -19,4 +18,4 @@ runs: - if: steps.install-cache.outputs.cache-hit != 'true' run: yarn install --frozen-lockfile --ignore-scripts --ignore-engines - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index beb8c3734c..95d2c4e754 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,13 +21,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/setup - run: yarn lint - + unit-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/setup - run: yarn test:ci diff --git a/api/market_data.js b/api/market_data.js index 9fe7fae94e..c1a681adb2 100644 --- a/api/market_data.js +++ b/api/market_data.js @@ -1,3 +1,5 @@ +import { kv } from "@vercel/kv"; + // Dia to Coingecko names const tickers = { "Tether USD": "tether", @@ -42,13 +44,24 @@ const dia_xlsd = { // retrieve Dia XLSD fair prices const fetchDiaXLSD = async () => { + const cache_key = "diaxlsd" + const cached = await kv.get(cache_key) + if (cached) { + return JSON.parse(cached) + } + const url = 'https://api.diadata.org/xlsd' const resp = await fetch(url, { headers: { "accept": "application/json" } }) if (!resp.ok) { throw new Error(resp.status) } const json = await resp.json() - return new Map(json.map(x => [x.Token, x])) + const result = new Map(json.map(x => [x.Token, x])) + + // cache the data for 120 seconds + await kv.set(cache_key, JSON.stringify(result), { ex: 120 }) + .catch(err => console.error('Unable to cache Dia data', err)) + return result; } const fetchDiaAsset = async (asset) => { @@ -81,14 +94,19 @@ const fetchDiaAsset = async (asset) => { } } } catch (error) { - console.log(error) + console.warn('Dia API error for asset: ', asset, error) throw error; } } const dia = async (args) => { - const assets = args.ids.split(',') + const cache_key = "dia_" + args.ids + const cached = await kv.get(cache_key) + if (cached) { + return JSON.parse(cached) + } + const assets = args.ids.split(',') return Promise .all(assets.map(x => fetchDiaAsset(x))) .then(x => x.reduce((map, obj) => { @@ -97,12 +115,34 @@ const dia = async (args) => { map[k] = obj[k] return map }, {})) + .then(async x => { + // cache the data for 120 seconds + kv.set(cache_key, JSON.stringify(x), { ex: 120 }) + .catch(err => console.error('Unable to cache Dia data', err)) + return x + }) } const coingecko = async (args) => { + const cache_key = "coingecko_" + args.ids + const cached = await kv.get(cache_key) + if (cached) { + console.log('Cached', cache_key, cached) + return JSON.parse(cached) + } + const url = 'https://api.coingecko.com/api/v3/simple/price?' + new URLSearchParams(args) const response = await fetch(url, { headers: { "accept": "application/json" } }) - return await response.json() + const data = await response.json() + if (!response.ok) { + throw new Error(data) + } + + // cache the data for 120 seconds + console.log('Caching', cache_key, JSON.stringify(data)) + kv.set(cache_key, JSON.stringify(data), { ex: 120 }) + .catch(err => console.error('Unable to cache coingecko data', err)) + return data; } const fetchPrices = (priceSource, args) => { @@ -111,12 +151,7 @@ const fetchPrices = (priceSource, args) => { } else if (priceSource === 'dia') { return dia(args) } else { - try { - return dia(args) - } catch (error) { - console.log(error) - return coingecko(args) - } + return dia(args).catch(() => coingecko(args)) } } @@ -124,10 +159,15 @@ export default async function (request, response) { const args = request.query const priceSource = args['price-source'] - const resp = await fetchPrices(priceSource, args) - return response - .status(200) - .setHeader("content-type", "application/json") - .setHeader("cache-control", "public, maxage=0, s-maxage=300") - .json(resp) + try { + const resp = await fetchPrices(priceSource, args) + return response + .status(200) + .setHeader("content-type", "application/json") + .setHeader("cache-control", "public, maxage=0, s-maxage=120") + .json(resp) + } catch (err) { + console.error('Unable to fetch prices', err) + return response.status(500); + } } diff --git a/api/package-lock.json b/api/package-lock.json index ed1e7be605..b16f648fb7 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -12,6 +12,7 @@ "@interlay/interbtc-api": "2.6.0", "@interlay/monetary-js": "0.7.3", "@polkadot/util-crypto": "12.6.1", + "@vercel/kv": "^1.0.1", "big.js": "6.1.1", "pg": "^8.10.0" }, @@ -778,12 +779,31 @@ "pg-types": "^2.2.0" } }, + "node_modules/@upstash/redis": { + "version": "1.25.1", + "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.25.1.tgz", + "integrity": "sha512-ACj0GhJ4qrQyBshwFgPod6XufVEfKX2wcaihsEvSdLYnY+m+pa13kGt1RXm/yTHKf4TQi/Dy2A8z/y6WUEOmlg==", + "dependencies": { + "crypto-js": "^4.2.0" + } + }, "node_modules/@vercel/build-utils": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-6.7.0.tgz", "integrity": "sha512-1cHgu0AzETSMLo1ugeHOT2pcrXNoZb1bwgxBP7yTIlJAKWLIEqPHbJWFvKQXZl2FV3kzzTctTrw/YNUWmntYiA==", "dev": true }, + "node_modules/@vercel/kv": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@vercel/kv/-/kv-1.0.1.tgz", + "integrity": "sha512-uTKddsqVYS2GRAM/QMNNXCTuw9N742mLoGRXoNDcyECaxEXvIHG0dEY+ZnYISV4Vz534VwJO+64fd9XeSggSKw==", + "dependencies": { + "@upstash/redis": "1.25.1" + }, + "engines": { + "node": ">=14.6" + } + }, "node_modules/@vercel/node": { "version": "2.10.2", "resolved": "https://registry.npmjs.org/@vercel/node/-/node-2.10.2.tgz", @@ -1175,6 +1195,11 @@ } } }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -3289,12 +3314,28 @@ "pg-types": "^2.2.0" } }, + "@upstash/redis": { + "version": "1.25.1", + "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.25.1.tgz", + "integrity": "sha512-ACj0GhJ4qrQyBshwFgPod6XufVEfKX2wcaihsEvSdLYnY+m+pa13kGt1RXm/yTHKf4TQi/Dy2A8z/y6WUEOmlg==", + "requires": { + "crypto-js": "^4.2.0" + } + }, "@vercel/build-utils": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-6.7.0.tgz", "integrity": "sha512-1cHgu0AzETSMLo1ugeHOT2pcrXNoZb1bwgxBP7yTIlJAKWLIEqPHbJWFvKQXZl2FV3kzzTctTrw/YNUWmntYiA==", "dev": true }, + "@vercel/kv": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@vercel/kv/-/kv-1.0.1.tgz", + "integrity": "sha512-uTKddsqVYS2GRAM/QMNNXCTuw9N742mLoGRXoNDcyECaxEXvIHG0dEY+ZnYISV4Vz534VwJO+64fd9XeSggSKw==", + "requires": { + "@upstash/redis": "1.25.1" + } + }, "@vercel/node": { "version": "2.10.2", "resolved": "https://registry.npmjs.org/@vercel/node/-/node-2.10.2.tgz", @@ -3626,6 +3667,11 @@ } } }, + "crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", diff --git a/api/package.json b/api/package.json index 8714c782b5..d6e3f5bf55 100644 --- a/api/package.json +++ b/api/package.json @@ -16,6 +16,7 @@ "@interlay/interbtc-api": "2.6.0", "@interlay/monetary-js": "0.7.3", "@polkadot/util-crypto": "12.6.1", + "@vercel/kv": "^1.0.1", "big.js": "6.1.1", "pg": "^8.10.0" } diff --git a/vercel.json b/vercel.json index 9413e0f124..a4a43c0c22 100644 --- a/vercel.json +++ b/vercel.json @@ -10,7 +10,7 @@ }, "api/market_data.js": { "memory": 128, - "maxDuration": 5 + "maxDuration": 10 }, "api/tvl_dex.js": { "memory": 256,