RegexMatch fix mongo

.gitignore

@@ -8,3 +8,4 @@ test/docker/config/*
 !test/docker/config/bitcoin.conf
 docker-commands.md
 .DS_Store
+query.txt

src/api/routes/dash.js

@@ -1,11 +1,30 @@
 const asyncHandler = require('express-async-handler')
 const router = require('express').Router()
 const { fromUnixTime, differenceInDays, parseISO, compareAsc, eachDayOfInterval, format } = require('date-fns')
-
+const debug = require('debug')('liquality:agent:dash')
 const Market = require('../../models/Market')
 const Order = require('../../models/Order')
 const MarketHistory = require('../../models/MarketHistory')
 
+const mergeJSON = (data) => {
+  const result = {}
+
+  data.forEach((dataSet) => {
+    for (let [key, value] of Object.entries(dataSet)) {
+      if (key === '_id' || key === 'wallet:count' || key === 'count') {
+        result[key] = value
+        continue
+      }
+      if (result[key]) {
+        result[key] += value
+      } else {
+        result[key] = value
+      }
+    }
+  })
+  return result
+}
+
 const addressHashVariants = (address) => {
   const addressLowerCase = address.toLowerCase()
 
@@ -244,59 +263,88 @@ router.get(
       ...req.query,
       status: ['AGENT_CLAIMED']
     })
-    const markets = (await Market.find({}, 'from to').lean().exec()).map((market) => `${market.from}-${market.to}`)
-
-    const $group = markets.reduce((acc, market) => {
-      acc[`market:${market}:sum:fromAmountUsd`] = {
-        $sum: { $cond: [{ $eq: ['$market', market] }, '$fromAmountUsd', 0] }
-      }
-      acc[`market:${market}:sum:toAmountUsd`] = {
-        $sum: { $cond: [{ $eq: ['$market', market] }, '$toAmountUsd', 0] }
-      }
-      acc[`market:${market}:count`] = {
-        $sum: { $cond: [{ $eq: ['$market', market] }, 1, 0] }
-      }
+    let allMarkets = (await Market.find({}, 'from to').lean().exec()).map((market) => `${market.from}-${market.to}`)
 
-      return acc
-    }, {})
+    const groupSize = 10
+    let result = []
+    const marketGroups = allMarkets
+      .map((e, i) => {
+        return i % groupSize === 0 ? allMarkets.slice(i, i + groupSize) : null
+      })
+      .filter((e) => {
+        return e
+      })
 
-    const result = await Order.aggregate([
-      {
-        $match: query
-      },
-      {
-        $addFields: {
-          market: { $concat: ['$from', '-', '$to'] },
-          date: { $dateToString: { format: '%Y-%m-%d', date: '$createdAt' } }
+    for (const markets of marketGroups) {
+      const $group = markets.reduce((acc, market) => {
+        acc[`market:${market}:sum:fromAmountUsd`] = {
+          $sum: { $cond: [{ $eq: ['$market', market] }, '$fromAmountUsd', 0] }
         }
-      },
-      {
-        $group: {
-          _id: '$date',
-          ...$group,
-          'wallet:sum:fromAmountUsd': {
-            $sum: {
-              $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, '$fromAmountUsd', 0]
-            }
-          },
-          'wallet:sum:toAmountUsd': {
-            $sum: {
-              $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, '$toAmountUsd', 0]
-            }
-          },
-          'wallet:count': {
-            $sum: {
-              $cond: [{ $regexMatch: { input: '$userAgent', regex: /^Wallet/i } }, 1, 0]
-            }
-          },
-          'sum:totalAgentFeeUsd': { $sum: '$totalAgentFeeUsd' },
-          'sum:totalUserFeeUsd': { $sum: '$totalUserFeeUsd' },
-          'sum:fromAmountUsd': { $sum: '$fromAmountUsd' },
-          'sum:toAmountUsd': { $sum: '$toAmountUsd' },
-          count: { $sum: 1 }
+        acc[`market:${market}:sum:toAmountUsd`] = {
+          $sum: { $cond: [{ $eq: ['$market', market] }, '$toAmountUsd', 0] }
+        }
+        acc[`market:${market}:count`] = {
+          $sum: { $cond: [{ $eq: ['$market', market] }, 1, 0] }
         }
-      }
-    ]).exec()
+
+        return acc
+      }, {})
+
+      const marketResult = await Order.aggregate([
+        {
+          $match: query
+        },
+        {
+          $addFields: {
+            market: { $concat: ['$from', '-', '$to'] },
+            date: { $dateToString: { format: '%Y-%m-%d', date: '$createdAt' } }
+          }
+        },
+        {
+          $group: {
+            _id: '$date',
+            ...$group,
+            'wallet:sum:fromAmountUsd': {
+              $sum: {
+                $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, '$fromAmountUsd', 0]
+              }
+            },
+            'wallet:sum:toAmountUsd': {
+              $sum: {
+                $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, '$toAmountUsd', 0]
+              }
+            },
+            'wallet:count': {
+              $sum: {
+                $cond: [{ $or: { input: '$userAgent', regex: /^Wallet/i } }, 1, 0]
+              }
+            },
+            'sum:totalAgentFeeUsd': { $sum: '$totalAgentFeeUsd' },
+            'sum:totalUserFeeUsd': { $sum: '$totalUserFeeUsd' },
+            'sum:fromAmountUsd': { $sum: '$fromAmountUsd' },
+            'sum:toAmountUsd': { $sum: '$toAmountUsd' },
+            count: { $sum: 1 }
+          }
+        }
+      ]).exec()
+      result.push(marketResult)
+    }
+
+    //flatten and group by date
+    let flattenResult = result.flat(1).reduce((r, ele) => {
+      r[ele._id] = r[ele._id] || []
+      r[ele._id].push(ele)
+      return r
+    }, Object.create(null))
+
+    debug('market result =>', flattenResult)
+    //merge all with sum
+    let finalResults = []
+    Object.keys(flattenResult).forEach((key) => {
+      finalResults.push(mergeJSON(flattenResult[key]))
+    })
+
+    result = finalResults
 
     const stats = result.map((json) => {
       json.date = json._id
@@ -319,7 +367,7 @@ router.get(
 
       return json
     })
-
+    debug('result=>', stats)
     const emptyDataPoint = {
       'wallet:sum:fromAmountUsd': 0,
       'wallet:sum:toAmountUsd': 0,
@@ -349,7 +397,7 @@ router.get(
     res.json({
       count: result.length,
       result: {
-        markets,
+        allMarkets,
         stats
       }
     })