api: add Vercel KV cache for storing market data (#1654)

* api: add Vercel KV cache for storing market data

* ci: update github actions

* api: increase market_data api timeout to 10 seconds

* api: increase market_data api timeout to 10 seconds

* api: add Vercel KV cache for storing market data

* api: add Vercel KV cache for storing market data

* api: add Vercel KV cache for storing market data - fix typo

* api: add Vercel KV cache for storing market data - fix typo

.github/actions/setup/action.yml

@@ -1,22 +1,21 @@
 name: Setup
-description: Sets up a Node.js environment, installs the dependencies using Yarn, 
+description: Sets up a Node.js environment, installs the dependencies using Yarn,
              and caches the installed dependencies for faster future builds.
 runs:
   using: composite
   steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 18
         registry-url: https://registry.npmjs.org
         cache: yarn
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       id: install-cache
       with:
         path: node_modules/
         key: ${{ runner.os }}-install-${{ hashFiles('**/yarn.lock') }}
 
     - if: steps.install-cache.outputs.cache-hit != 'true'
       run: yarn install --frozen-lockfile --ignore-scripts --ignore-engines
-      shell: bash
+      shell: bash

.github/workflows/test.yml

@@ -21,13 +21,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: yarn lint
-  
+
   unit-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: yarn test:ci

api/market_data.js

@@ -1,3 +1,5 @@
+import { kv } from "@vercel/kv";
+
 // Dia to Coingecko names
 const tickers = {
   "Tether USD": "tether",
@@ -42,13 +44,24 @@ const dia_xlsd = {
 
 // retrieve Dia XLSD fair prices
 const fetchDiaXLSD = async () => {
+  const cache_key = "diaxlsd"
+  const cached = await kv.get(cache_key)
+  if (cached) {
+    return JSON.parse(cached)
+  }
+
   const url = 'https://api.diadata.org/xlsd'
   const resp = await fetch(url, { headers: { "accept": "application/json" } })
   if (!resp.ok) {
     throw new Error(resp.status)
   }
   const json = await resp.json()
-  return new Map(json.map(x => [x.Token, x]))
+  const result = new Map(json.map(x => [x.Token, x]))
+
+  // cache the data for 120 seconds
+  await kv.set(cache_key, JSON.stringify(result), { ex: 120 })
+    .catch(err => console.error('Unable to cache Dia data', err))
+  return result;
 }
 
 const fetchDiaAsset = async (asset) => {
@@ -81,14 +94,19 @@ const fetchDiaAsset = async (asset) => {
       }
     }
   } catch (error) {
-    console.log(error)
+    console.warn('Dia API error for asset: ', asset, error)
     throw error;
   }
 }
 
 const dia = async (args) => {
-  const assets = args.ids.split(',')
+  const cache_key = "dia_" + args.ids
+  const cached = await kv.get(cache_key)
+  if (cached) {
+    return JSON.parse(cached)
+  }
 
+  const assets = args.ids.split(',')
   return Promise
     .all(assets.map(x => fetchDiaAsset(x)))
     .then(x => x.reduce((map, obj) => {
@@ -97,12 +115,34 @@ const dia = async (args) => {
       map[k] = obj[k]
       return map
     }, {}))
+    .then(async x => {
+      // cache the data for 120 seconds
+      kv.set(cache_key, JSON.stringify(x), { ex: 120 })
+        .catch(err => console.error('Unable to cache Dia data', err))
+      return x
+    })
 }
 
 const coingecko = async (args) => {
+  const cache_key = "coingecko_" + args.ids
+  const cached = await kv.get(cache_key)
+  if (cached) {
+    console.log('Cached', cache_key, cached)
+    return JSON.parse(cached)
+  }
+
   const url = 'https://api.coingecko.com/api/v3/simple/price?' + new URLSearchParams(args)
   const response = await fetch(url, { headers: { "accept": "application/json" } })
-  return await response.json()
+  const data = await response.json()
+  if (!response.ok) {
+    throw new Error(data)
+  }
+
+  // cache the data for 120 seconds
+  console.log('Caching', cache_key, JSON.stringify(data))
+  kv.set(cache_key, JSON.stringify(data), { ex: 120 })
+    .catch(err => console.error('Unable to cache coingecko data', err))
+  return data;
 }
 
 const fetchPrices = (priceSource, args) => {
@@ -111,23 +151,23 @@ const fetchPrices = (priceSource, args) => {
   } else if (priceSource === 'dia') {
     return dia(args)
   } else {
-    try {
-      return dia(args)
-    } catch (error) {
-      console.log(error)
-      return coingecko(args)
-    }
+    return dia(args).catch(() => coingecko(args))
   }
 }
 
 export default async function (request, response) {
   const args = request.query
   const priceSource = args['price-source']
 
-  const resp = await fetchPrices(priceSource, args)
-  return response
-    .status(200)
-    .setHeader("content-type", "application/json")
-    .setHeader("cache-control", "public, maxage=0, s-maxage=300")
-    .json(resp)
+  try {
+    const resp = await fetchPrices(priceSource, args)
+    return response
+      .status(200)
+      .setHeader("content-type", "application/json")
+      .setHeader("cache-control", "public, maxage=0, s-maxage=120")
+      .json(resp)
+  } catch (err) {
+    console.error('Unable to fetch prices', err)
+    return response.status(500);
+  }
 }

api/package.json

@@ -16,6 +16,7 @@
     "@interlay/interbtc-api": "2.6.0",
     "@interlay/monetary-js": "0.7.3",
     "@polkadot/util-crypto": "12.6.1",
+    "@vercel/kv": "^1.0.1",
     "big.js": "6.1.1",
     "pg": "^8.10.0"
   }

vercel.json

@@ -10,7 +10,7 @@
     },
     "api/market_data.js": {
       "memory": 128,
-      "maxDuration": 5
+      "maxDuration": 10
     },
     "api/tvl_dex.js": {
       "memory": 256,