From 312b2a75f1460e3b832f454f9c6c0aac48799ee5 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Tue, 25 Jul 2023 22:30:34 +0200 Subject: [PATCH 1/7] Initial address endpoint feature commit, add addresses endpoint, start implementation on getAddress function (implement total_balance, locked_balance, transferrable_balance) --- src/app.ts | 7 +++++ src/get-addresses.ts | 13 +++++++++ src/joyApi.ts | 65 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/get-addresses.ts diff --git a/src/app.ts b/src/app.ts index fea25d0..def1fbf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -12,6 +12,7 @@ import getPrice from "./get-price"; import getCirculatingSupply from "./get-circulating-supply"; import { calculateSecondsUntilNext5MinuteInterval } from "./utils"; import getTotalSupply from "./get-total-supply"; +import getAddresses from "./get-addresses"; const app = express(); const cache = apicache.middleware; @@ -56,6 +57,12 @@ app.get("/", cache("1 hour"), async (req, res) => { res.send(status); }); +app.get("/addresses", async (req, res) => { + const addresses = await getAddresses(); + res.setHeader("Content-Type", "application/json"); + res.send({ message: addresses }); +}); + app.get("/budgets", cache("1 day"), async (req, res) => { let budgets = await getBudgets(); res.setHeader("Content-Type", "application/json"); diff --git a/src/get-addresses.ts b/src/get-addresses.ts new file mode 100644 index 0000000..b355382 --- /dev/null +++ b/src/get-addresses.ts @@ -0,0 +1,13 @@ +import { JoyApi } from "./joyApi"; + +const api = new JoyApi(); + +const getAddresses = async () => { + await api.init; + + const addresses = await api.getAddresses(); + + return { addresses }; +}; + +export default getAddresses; diff --git a/src/joyApi.ts b/src/joyApi.ts index c9e9448..8050739 100644 --- a/src/joyApi.ts +++ b/src/joyApi.ts @@ -21,6 +21,7 @@ if(process.env.QUERY_NODE === undefined){ throw new Error("Missing QUERY_NODE in .env!"); } const QUERY_NODE = process.env.QUERY_NODE; +const VESTING_STRING_HEX = "0x76657374696e6720"; type SystemData = { chain: string @@ -373,8 +374,6 @@ export class JoyApi { async calculateCirculatingSupply() { - const VESTING_STRING_HEX = "0x76657374696e6720"; - const accounts = []; const amounts: BN[] = []; const lockData = await this.api.query.balances.locks.entries(); @@ -407,6 +406,68 @@ export class JoyApi { return totalSupply - this.toJOY(total); } + async getAddresses() { + const lockData = await this.api.query.balances.locks.entries(); + const lockDataAdresses: any[] = []; + const resultData: { + [key: string]: { + tempAmount: BN; + total_balance: number; + transferrable_balance: number; + locked_balance: number; + vesting_lock: number; + }; + } = {}; + + for (let [storageKey, palletBalances] of lockData) { + let biggestLock = new BN(0); + let vestingLock = new BN(0); + const address = storageKey.args[0].toString(); + + lockDataAdresses.push(address); + resultData[address] = { + tempAmount: new BN(0), + total_balance: 0, + transferrable_balance: 0, + locked_balance: 0, + vesting_lock: 0, + }; + + for (let palletBalance of palletBalances) { + if (palletBalance.amount.toBn().gt(biggestLock)) { + biggestLock = palletBalance.amount.toBn(); + + if ( + palletBalance.id.toString() === VESTING_STRING_HEX && + palletBalance.amount.toBn().gt(vestingLock) + ) { + vestingLock = palletBalance.amount.toBn(); + } + } + } + + if (biggestLock.gt(new BN(0))) { + resultData[address].tempAmount = biggestLock; + resultData[address].vesting_lock = this.toJOY(vestingLock); + } + } + + const intAccs = await this.api.query.system.account.multi(lockDataAdresses); + + intAccs.forEach((val, index) => { + const address = lockDataAdresses[index]; + const totalBalance = this.toJOY(val.data.free); + const lockedBalance = this.toJOY( + BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) + ); + resultData[address].total_balance = totalBalance; + resultData[address].locked_balance = lockedBalance; + resultData[address].transferrable_balance = totalBalance - lockedBalance; + }); + + return resultData; + } + protected async fetchNetworkStatus(): Promise { const [ [ From 8f9131b86f00fe9ac20c452cb7f3069a74c4e419 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Thu, 27 Jul 2023 00:33:48 +0200 Subject: [PATCH 2/7] Implement vestable information, remove unnecessary temp data --- src/joyApi.ts | 58 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/joyApi.ts b/src/joyApi.ts index 8050739..790566c 100644 --- a/src/joyApi.ts +++ b/src/joyApi.ts @@ -82,6 +82,14 @@ type NetworkStatus = { runtimeData: RuntimeData } +type Address = { + total_balance: number + transferrable_balance: number + locked_balance: number + vesting_lock: number + vestable: number +} + export class JoyApi { endpoint: string; tokenDecimals!: number; @@ -407,21 +415,19 @@ export class JoyApi { } async getAddresses() { + // TODO: Is this the best way to get the current block? + const currentBlock = (await this.api.derive.chain.bestNumber()).toNumber(); const lockData = await this.api.query.balances.locks.entries(); const lockDataAdresses: any[] = []; const resultData: { [key: string]: { tempAmount: BN; - total_balance: number; - transferrable_balance: number; - locked_balance: number; - vesting_lock: number; - }; + } & Address; } = {}; for (let [storageKey, palletBalances] of lockData) { let biggestLock = new BN(0); - let vestingLock = new BN(0); + let biggestVestingLock = new BN(0); const address = storageKey.args[0].toString(); lockDataAdresses.push(address); @@ -431,6 +437,7 @@ export class JoyApi { transferrable_balance: 0, locked_balance: 0, vesting_lock: 0, + vestable: 0, }; for (let palletBalance of palletBalances) { @@ -439,23 +446,52 @@ export class JoyApi { if ( palletBalance.id.toString() === VESTING_STRING_HEX && - palletBalance.amount.toBn().gt(vestingLock) + palletBalance.amount.toBn().gt(biggestVestingLock) ) { - vestingLock = palletBalance.amount.toBn(); + biggestVestingLock = palletBalance.amount.toBn(); } } } if (biggestLock.gt(new BN(0))) { resultData[address].tempAmount = biggestLock; - resultData[address].vesting_lock = this.toJOY(vestingLock); + resultData[address].vesting_lock = this.toJOY(biggestVestingLock); } } const intAccs = await this.api.query.system.account.multi(lockDataAdresses); + const vestingData = await this.api.query.vesting.vesting.multi(lockDataAdresses); intAccs.forEach((val, index) => { const address = lockDataAdresses[index]; + const currentAddressVestingData = vestingData[index]; + const currentAddressVestingEntries = currentAddressVestingData.unwrapOr(null); + + if (currentAddressVestingEntries !== null) { + const largestVestingEntry = currentAddressVestingEntries.reduce((accumulator, val) => { + // We can add the following line to mitigate problem but it's not a fix. + // && val.startingBlock.eq(new BN(0)) && val.perBlock.lt(new BN(1_000_000_000_000)) + if (val.locked.gt(accumulator.locked)) { + return val; + } + + return accumulator; + }); + + const maxFromOriginVestable = largestVestingEntry.perBlock.mul(new BN(currentBlock)); + const maxRemainingVestable = largestVestingEntry.locked.sub(this.toHAPI(resultData[address].vesting_lock)); + const currentlyVestable = maxFromOriginVestable.sub(maxRemainingVestable); + + // TODO: Due to a massive vesting lock the following line will end up with a massive negative value. + // We catch that for now but that should be found and replaced with the actual vesting lock earlier. + try { + resultData[address].vestable = this.toJOY(currentlyVestable); + } catch (e) { + console.log(`Vestable calculation error. At value: ${currentlyVestable.toString()}`) + console.log(`At address: ${address}`); + } + } + const totalBalance = this.toJOY(val.data.free); const lockedBalance = this.toJOY( BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) @@ -463,9 +499,11 @@ export class JoyApi { resultData[address].total_balance = totalBalance; resultData[address].locked_balance = lockedBalance; resultData[address].transferrable_balance = totalBalance - lockedBalance; + + delete (resultData[address] as any).tempAmount; }); - return resultData; + return resultData as { [key: string]: Address }; } protected async fetchNetworkStatus(): Promise { From 52b0fbba0bf9807426a9fde79335222ec722eff6 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Mon, 31 Jul 2023 22:29:07 +0200 Subject: [PATCH 3/7] Initial fixes (WIP) --- src/joyApi.ts | 131 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 46 deletions(-) diff --git a/src/joyApi.ts b/src/joyApi.ts index 790566c..d46d827 100644 --- a/src/joyApi.ts +++ b/src/joyApi.ts @@ -415,9 +415,12 @@ export class JoyApi { } async getAddresses() { - // TODO: Is this the best way to get the current block? - const currentBlock = (await this.api.derive.chain.bestNumber()).toNumber(); - const lockData = await this.api.query.balances.locks.entries(); + const finalizedHeadHash = await this.finalizedHash(); + const { number: blockNumber } = await this.api.rpc.chain.getHeader(`${finalizedHeadHash}`); + const finalizedApi = await this.api.at(finalizedHeadHash); + const currentBlock = blockNumber.toBn(); + + const lockData = await finalizedApi.query.balances.locks.entries(); const lockDataAdresses: any[] = []; const resultData: { [key: string]: { @@ -430,7 +433,6 @@ export class JoyApi { let biggestVestingLock = new BN(0); const address = storageKey.args[0].toString(); - lockDataAdresses.push(address); resultData[address] = { tempAmount: new BN(0), total_balance: 0, @@ -441,68 +443,105 @@ export class JoyApi { }; for (let palletBalance of palletBalances) { - if (palletBalance.amount.toBn().gt(biggestLock)) { + if(palletBalance.amount.toBn().gt(biggestLock)) { biggestLock = palletBalance.amount.toBn(); + } - if ( - palletBalance.id.toString() === VESTING_STRING_HEX && - palletBalance.amount.toBn().gt(biggestVestingLock) - ) { - biggestVestingLock = palletBalance.amount.toBn(); - } + if ( + palletBalance.id.toString() === VESTING_STRING_HEX && + palletBalance.amount.toBn().gt(biggestVestingLock) + ) { + biggestVestingLock = palletBalance.amount.toBn(); } } - if (biggestLock.gt(new BN(0))) { + if(biggestLock.gt(new BN(0))) { resultData[address].tempAmount = biggestLock; + } + + if (biggestVestingLock.gt(new BN(0))) { + lockDataAdresses.push(address); resultData[address].vesting_lock = this.toJOY(biggestVestingLock); } } - const intAccs = await this.api.query.system.account.multi(lockDataAdresses); - const vestingData = await this.api.query.vesting.vesting.multi(lockDataAdresses); + const intAccs = await finalizedApi.query.system.account.multi(lockDataAdresses); + const vestingData = await finalizedApi.query.vesting.vesting.multi(lockDataAdresses); intAccs.forEach((val, index) => { const address = lockDataAdresses[index]; const currentAddressVestingData = vestingData[index]; const currentAddressVestingEntries = currentAddressVestingData.unwrapOr(null); - if (currentAddressVestingEntries !== null) { - const largestVestingEntry = currentAddressVestingEntries.reduce((accumulator, val) => { - // We can add the following line to mitigate problem but it's not a fix. - // && val.startingBlock.eq(new BN(0)) && val.perBlock.lt(new BN(1_000_000_000_000)) - if (val.locked.gt(accumulator.locked)) { - return val; - } - - return accumulator; - }); - - const maxFromOriginVestable = largestVestingEntry.perBlock.mul(new BN(currentBlock)); - const maxRemainingVestable = largestVestingEntry.locked.sub(this.toHAPI(resultData[address].vesting_lock)); - const currentlyVestable = maxFromOriginVestable.sub(maxRemainingVestable); - - // TODO: Due to a massive vesting lock the following line will end up with a massive negative value. - // We catch that for now but that should be found and replaced with the actual vesting lock earlier. - try { - resultData[address].vestable = this.toJOY(currentlyVestable); - } catch (e) { - console.log(`Vestable calculation error. At value: ${currentlyVestable.toString()}`) - console.log(`At address: ${address}`); + if(currentAddressVestingEntries !== null) { + const [vestingSum, vestable] = currentAddressVestingEntries.reduce(([vestingSumAcc, vestableAcc], vestingEntry) => { + const maxFromOriginVestable = vestingEntry.perBlock.mul(currentBlock); + const maxRemainingVestable = vestingEntry.locked.sub(this.toHAPI(resultData[address].vesting_lock)); + const currentlyVestable = maxFromOriginVestable.sub(maxRemainingVestable); + + return [vestingSumAcc.add(vestingEntry.locked), vestableAcc.add(currentlyVestable)]; + }, [new BN(0), new BN(0)]); + + const totalBalance = this.toJOY(val.data.free); + const lockedBalance = this.toJOY( + BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) + ); + resultData[address].total_balance = totalBalance; + + if(vestingSum.lte(val.data.free)) { + // const unlockedAmount = val.data.free.sub(vestingSum); + // const lockedAmount = vestingSum; + + resultData[address].transferrable_balance = totalBalance - lockedBalance; + resultData[address].locked_balance = lockedBalance; + resultData[address].vestable = this.toJOY(vestable); + } else { + resultData[address].locked_balance = this.toJOY(val.data.free); } } - - const totalBalance = this.toJOY(val.data.free); - const lockedBalance = this.toJOY( - BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) - ); - resultData[address].total_balance = totalBalance; - resultData[address].locked_balance = lockedBalance; - resultData[address].transferrable_balance = totalBalance - lockedBalance; - - delete (resultData[address] as any).tempAmount; }); + // intAccs.forEach((val, index) => { + // const address = lockDataAdresses[index]; + // const currentAddressVestingData = vestingData[index]; + // const currentAddressVestingEntries = currentAddressVestingData.unwrapOr(null); + + // if (currentAddressVestingEntries !== null) { + // const largestVestingEntry = currentAddressVestingEntries.reduce((accumulator, val) => { + // // We can add the following line to mitigate problem but it's not a fix. + // // && val.startingBlock.eq(new BN(0)) && val.perBlock.lt(new BN(1_000_000_000_000)) + // if (val.locked.gt(accumulator.locked)) { + // return val; + // } + + // return accumulator; + // }); + + // const maxFromOriginVestable = largestVestingEntry.perBlock.mul(currentBlock); + // const maxRemainingVestable = largestVestingEntry.locked.sub(this.toHAPI(resultData[address].vesting_lock)); + // const currentlyVestable = maxFromOriginVestable.sub(maxRemainingVestable); + + // // TODO: Due to a massive vesting lock the following line will end up with a massive negative value. + // // We catch that for now but that should be found and replaced with the actual vesting lock earlier. + // try { + // resultData[address].vestable = this.toJOY(currentlyVestable); + // } catch (e) { + // console.log(`Vestable calculation error. At value: ${currentlyVestable.toString()}`) + // console.log(`At address: ${address}`); + // } + // } + + // const totalBalance = this.toJOY(val.data.free); + // const lockedBalance = this.toJOY( + // BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) + // ); + // resultData[address].total_balance = totalBalance; + // resultData[address].locked_balance = lockedBalance; + // resultData[address].transferrable_balance = totalBalance - lockedBalance; + + // delete (resultData[address] as any).tempAmount; + // }); + return resultData as { [key: string]: Address }; } From 6987f6228297d689a607f4a93bcbb343ed76f184 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Tue, 1 Aug 2023 02:44:34 +0200 Subject: [PATCH 4/7] Install and setup ejs, initial work on address_ui endpoint --- package.json | 1 + public/address_ui.ejs | 39 ++++++++++++++++++++++++++++++++ src/app.ts | 20 +++++++++++++++++ yarn.lock | 52 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 public/address_ui.ejs diff --git a/package.json b/package.json index 29ea774..091a86f 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "cors": "^2.8.5", "cross-fetch": "^3.0.6", "dotenv": "^8.2.0", + "ejs": "^3.1.9", "express": "^4.17.1", "inquirer": "^7.3.3", "locks": "^0.2.2", diff --git a/public/address_ui.ejs b/public/address_ui.ejs new file mode 100644 index 0000000..f94dae7 --- /dev/null +++ b/public/address_ui.ejs @@ -0,0 +1,39 @@ + + + + + + + Joystream Address UI + + + + +
+
+
+
+ + +
+
+

Recorded at block: <%= locals?.recordedAtBlock %>

+

Recorded at time: <%= locals?.recordedAtTime %>

+

Locked balance: <%= locals?.lockedBalance %>

+

Total balance: <%= locals?.totalBalance %>

+

Transferrable balance: <%= locals?.transferrableBalance %>

+

Vesting lock: <%= locals?.vestingLock %>

+

Vestable: <%= locals?.vestable %>

+
+
+ +
+
+
+
+ + \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index def1fbf..045d651 100644 --- a/src/app.ts +++ b/src/app.ts @@ -20,9 +20,11 @@ const port = process.env.PORT || 8081; const CAROUSEL_DATA_PATH = path.join(__dirname, "../carousel-data.json"); const CIRCULATING_SUPPLY_DATA_PATH = path.join(__dirname, "../circulating-supply-data.json"); const TOTAL_SUPPLY_DATA_PATH = path.join(__dirname, "../total-supply-data.json"); +const ADDRESS_UI_HTML = path.join(__dirname, "../public/address_ui.ejs"); app.use(cors()); app.use(express.json()); +app.set("view engine", "ejs"); const scheduleCronJob = async () => { console.log("Scheduling cron job..."); @@ -63,6 +65,24 @@ app.get("/addresses", async (req, res) => { res.send({ message: addresses }); }); +app.get("/address_ui", async (req, res) => { + if(!req.query.address) { + res.render(ADDRESS_UI_HTML, { message: undefined}); + return; + } + + res.render(ADDRESS_UI_HTML, { + recordedAtBlock: 3_365_488, + recordedAtTime: new Date().toISOString(), + address: req.query.address, + lockedBalance: 81501.99, + totalBalance: 481502.016, + transferrableBalance: 0.0195, + vestingLock: 237404.53, + vestable:2858.22 + }); +}); + app.get("/budgets", cache("1 day"), async (req, res) => { let budgets = await getBudgets(); res.setHeader("Content-Type", "application/json"); diff --git a/yarn.lock b/yarn.lock index ea3dcaf..def725a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,11 @@ async@^2.6.3, async@~2.6.1: dependencies: lodash "^4.17.14" +async@^3.2.3: + version "3.2.4" + resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + async@~3.2.0: version "3.2.3" resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" @@ -908,6 +913,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -966,7 +978,7 @@ chalk@3.0.0, chalk@~3.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.1.0: +chalk@^4.0.2, chalk@^4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -1274,6 +1286,13 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= +ejs@^3.1.9: + version "3.1.9" + resolved "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361" + integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ== + dependencies: + jake "^10.8.5" + emitter-listener@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8" @@ -1487,6 +1506,13 @@ file-uri-to-path@2: resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-2.0.0.tgz#7b415aeba227d575851e0a5b0c640d7656403fba" integrity sha512-hjPFI8oE/2iQPVe4gbrJ73Pp+Xfub2+WI2LlXDbsaJBwT5wuMh35WNWVYYTpnz895shtwfyutMFLFywpQAFdLg== +filelist@^1.0.4: + version "1.0.4" + resolved "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz#f78978a1e944775ff9e62e744424f215e58352b5" + integrity sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q== + dependencies: + minimatch "^5.0.1" + fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -1891,6 +1917,16 @@ isarray@0.0.1: resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= +jake@^10.8.5: + version "10.8.7" + resolved "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz#63a32821177940c33f356e0ba44ff9d34e1c7d8f" + integrity sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w== + dependencies: + async "^3.2.3" + chalk "^4.0.2" + filelist "^1.0.4" + minimatch "^3.1.2" + js-git@^0.7.8: version "0.7.8" resolved "https://registry.yarnpkg.com/js-git/-/js-git-0.7.8.tgz#52fa655ab61877d6f1079efc6534b554f31e5444" @@ -2061,6 +2097,20 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +minimatch@^3.1.2: + version "3.1.2" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimatch@^5.0.1: + version "5.1.6" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" From b1caf87a0b5ac24772621932bef25e8416afa0cd Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Wed, 2 Aug 2023 23:46:25 +0200 Subject: [PATCH 5/7] Add addressd data fetching to cron job, add address datafile name to gitignore, add /addresses, /address and /address_ui routes, make small changes to getAddresses function --- .gitignore | 3 +- public/address_ui.ejs | 12 ++--- src/app.ts | 101 +++++++++++++++++++++++++++++++----------- src/joyApi.ts | 55 ++++------------------- 4 files changed, 92 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 85b1ada..5ab0eb1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ out.log exchanges.json carousel-data.json circulating-supply-data.json -total-supply-data.json \ No newline at end of file +total-supply-data.json +addresses-data.json \ No newline at end of file diff --git a/public/address_ui.ejs b/public/address_ui.ejs index f94dae7..0ee560d 100644 --- a/public/address_ui.ejs +++ b/public/address_ui.ejs @@ -19,12 +19,12 @@
-

Recorded at block: <%= locals?.recordedAtBlock %>

-

Recorded at time: <%= locals?.recordedAtTime %>

-

Locked balance: <%= locals?.lockedBalance %>

-

Total balance: <%= locals?.totalBalance %>

-

Transferrable balance: <%= locals?.transferrableBalance %>

-

Vesting lock: <%= locals?.vestingLock %>

+

Recorded at block: <%= locals?.recorded_at_block %>

+

Recorded at time: <%= locals?.recorded_at_time %>

+

Locked balance: <%= locals?.locked_balance %>

+

Total balance: <%= locals?.total_balance %>

+

Transferrable balance: <%= locals?.transferrable_balance %>

+

Vesting lock: <%= locals?.vesting_lock %>

Vestable: <%= locals?.vestable %>

diff --git a/src/app.ts b/src/app.ts index 045d651..af44aea 100644 --- a/src/app.ts +++ b/src/app.ts @@ -20,6 +20,7 @@ const port = process.env.PORT || 8081; const CAROUSEL_DATA_PATH = path.join(__dirname, "../carousel-data.json"); const CIRCULATING_SUPPLY_DATA_PATH = path.join(__dirname, "../circulating-supply-data.json"); const TOTAL_SUPPLY_DATA_PATH = path.join(__dirname, "../total-supply-data.json"); +const ADDRESSES_DATA_PATH = path.join(__dirname, "../addresses-data.json"); const ADDRESS_UI_HTML = path.join(__dirname, "../public/address_ui.ejs"); app.use(cors()); @@ -38,14 +39,31 @@ const scheduleCronJob = async () => { const fetchAndWriteSupplyData = async () => { const circulatingSupplyData = await getCirculatingSupply(); const totalSupplyData = await getTotalSupply(); + const { addresses } = await getAddresses(); fs.writeFileSync(CIRCULATING_SUPPLY_DATA_PATH, JSON.stringify(circulatingSupplyData, null, 2)); fs.writeFileSync(TOTAL_SUPPLY_DATA_PATH, JSON.stringify(totalSupplyData, null, 2)); + fs.writeFileSync( + ADDRESSES_DATA_PATH, + JSON.stringify( + { + total_supply: totalSupplyData.totalSupply, + circulating_supply: circulatingSupplyData.circulatingSupply, + addresses, + }, + null, + 2 + ) + ); }; // Fetch data initially such that we have something to serve. There will at most // be a buffer of 5 minutes from this running until the first cron execution. - if (!fs.existsSync(CIRCULATING_SUPPLY_DATA_PATH) || !fs.existsSync(TOTAL_SUPPLY_DATA_PATH)) + if ( + !fs.existsSync(CIRCULATING_SUPPLY_DATA_PATH) || + !fs.existsSync(TOTAL_SUPPLY_DATA_PATH) || + !fs.existsSync(ADDRESSES_DATA_PATH) + ) await fetchAndWriteSupplyData(); if (!fs.existsSync(CAROUSEL_DATA_PATH)) await fetchAndWriteCarouselData(); @@ -59,30 +77,6 @@ app.get("/", cache("1 hour"), async (req, res) => { res.send(status); }); -app.get("/addresses", async (req, res) => { - const addresses = await getAddresses(); - res.setHeader("Content-Type", "application/json"); - res.send({ message: addresses }); -}); - -app.get("/address_ui", async (req, res) => { - if(!req.query.address) { - res.render(ADDRESS_UI_HTML, { message: undefined}); - return; - } - - res.render(ADDRESS_UI_HTML, { - recordedAtBlock: 3_365_488, - recordedAtTime: new Date().toISOString(), - address: req.query.address, - lockedBalance: 81501.99, - totalBalance: 481502.016, - transferrableBalance: 0.0195, - vestingLock: 237404.53, - vestable:2858.22 - }); -}); - app.get("/budgets", cache("1 day"), async (req, res) => { let budgets = await getBudgets(); res.setHeader("Content-Type", "application/json"); @@ -136,6 +130,63 @@ app.get("/total-supply", async (req, res) => { res.status(503).send(); }); +app.get("/addresses", async (req, res) => { + if (fs.existsSync(ADDRESSES_DATA_PATH)) { + const addressesFileData = fs.readFileSync(ADDRESSES_DATA_PATH); + res.setHeader("Content-Type", "application/json"); + const addressesData = JSON.parse(addressesFileData.toString()); + res.send(addressesData); + + return; + } + + res.setHeader("Retry-After", calculateSecondsUntilNext5MinuteInterval()); + res.status(503).send(); +}); + +app.get("/address", async (req, res) => { + if (fs.existsSync(ADDRESSES_DATA_PATH)) { + res.setHeader("Content-Type", "application/json"); + + if (!req.query.address) { + res.send({}); + return; + } + + const addressesFileData = fs.readFileSync(ADDRESSES_DATA_PATH); + const { addresses } = JSON.parse(addressesFileData.toString()); + const receivedAddress = req.query.address as string; + + res.send({ [receivedAddress]: addresses[receivedAddress] }); + + return; + } + + res.setHeader("Retry-After", calculateSecondsUntilNext5MinuteInterval()); + res.status(503).send(); +}); + +app.get("/address_ui", async (req, res) => { + if (!fs.existsSync(ADDRESSES_DATA_PATH)) { + res.setHeader("Retry-After", calculateSecondsUntilNext5MinuteInterval()); + res.status(503).send(); + return; + } + + if (!req.query.address) { + res.render(ADDRESS_UI_HTML); + return; + } + + const addressesFileData = fs.readFileSync(ADDRESSES_DATA_PATH); + const { addresses } = JSON.parse(addressesFileData.toString()); + + res.render(ADDRESS_UI_HTML, { + address: req.query.address, + ...addresses[req.query.address as string], + }); +}); + scheduleCronJob().then(() => { app.listen(port, () => { log(`server started at http://localhost:${port}`); diff --git a/src/joyApi.ts b/src/joyApi.ts index d46d827..57112d4 100644 --- a/src/joyApi.ts +++ b/src/joyApi.ts @@ -83,6 +83,8 @@ type NetworkStatus = { } type Address = { + recorded_at_block: number; + recorded_at_time: string; total_balance: number transferrable_balance: number locked_balance: number @@ -417,6 +419,7 @@ export class JoyApi { async getAddresses() { const finalizedHeadHash = await this.finalizedHash(); const { number: blockNumber } = await this.api.rpc.chain.getHeader(`${finalizedHeadHash}`); + const timestamp = await this.api.query.timestamp.now.at(finalizedHeadHash); const finalizedApi = await this.api.at(finalizedHeadHash); const currentBlock = blockNumber.toBn(); @@ -435,6 +438,8 @@ export class JoyApi { resultData[address] = { tempAmount: new BN(0), + recorded_at_block: currentBlock.toNumber(), + recorded_at_time: (new Date(timestamp.toNumber())).toISOString(), total_balance: 0, transferrable_balance: 0, locked_balance: 0, @@ -487,60 +492,16 @@ export class JoyApi { BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) ); resultData[address].total_balance = totalBalance; + resultData[address].transferrable_balance = totalBalance - lockedBalance; + resultData[address].locked_balance = lockedBalance; if(vestingSum.lte(val.data.free)) { - // const unlockedAmount = val.data.free.sub(vestingSum); - // const lockedAmount = vestingSum; - - resultData[address].transferrable_balance = totalBalance - lockedBalance; - resultData[address].locked_balance = lockedBalance; resultData[address].vestable = this.toJOY(vestable); - } else { - resultData[address].locked_balance = this.toJOY(val.data.free); } } }); - // intAccs.forEach((val, index) => { - // const address = lockDataAdresses[index]; - // const currentAddressVestingData = vestingData[index]; - // const currentAddressVestingEntries = currentAddressVestingData.unwrapOr(null); - - // if (currentAddressVestingEntries !== null) { - // const largestVestingEntry = currentAddressVestingEntries.reduce((accumulator, val) => { - // // We can add the following line to mitigate problem but it's not a fix. - // // && val.startingBlock.eq(new BN(0)) && val.perBlock.lt(new BN(1_000_000_000_000)) - // if (val.locked.gt(accumulator.locked)) { - // return val; - // } - - // return accumulator; - // }); - - // const maxFromOriginVestable = largestVestingEntry.perBlock.mul(currentBlock); - // const maxRemainingVestable = largestVestingEntry.locked.sub(this.toHAPI(resultData[address].vesting_lock)); - // const currentlyVestable = maxFromOriginVestable.sub(maxRemainingVestable); - - // // TODO: Due to a massive vesting lock the following line will end up with a massive negative value. - // // We catch that for now but that should be found and replaced with the actual vesting lock earlier. - // try { - // resultData[address].vestable = this.toJOY(currentlyVestable); - // } catch (e) { - // console.log(`Vestable calculation error. At value: ${currentlyVestable.toString()}`) - // console.log(`At address: ${address}`); - // } - // } - - // const totalBalance = this.toJOY(val.data.free); - // const lockedBalance = this.toJOY( - // BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) - // ); - // resultData[address].total_balance = totalBalance; - // resultData[address].locked_balance = lockedBalance; - // resultData[address].transferrable_balance = totalBalance - lockedBalance; - - // delete (resultData[address] as any).tempAmount; - // }); + Object.keys(resultData).forEach((address) => { delete (resultData[address] as any).tempAmount; }); return resultData as { [key: string]: Address }; } From 8d28ce9f3cc17089b032517b6ac10cb26149b840 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Sat, 5 Aug 2023 01:28:30 +0200 Subject: [PATCH 6/7] Initial review fixes, take into account all balances (not just ones with locks), fix calculation of balances due to misplaced logic --- src/joyApi.ts | 56 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/src/joyApi.ts b/src/joyApi.ts index 57112d4..ff04d48 100644 --- a/src/joyApi.ts +++ b/src/joyApi.ts @@ -9,7 +9,8 @@ import { PalletWorkingGroupGroupWorker as Worker, PalletReferendumReferendumStage as ReferendumStage, PalletCouncilCouncilStageUpdate as CouncilStageUpdate, - PalletVestingVestingInfo + PalletVestingVestingInfo, + FrameSystemAccountInfo } from '@polkadot/types/lookup' import { Vec } from '@polkadot/types'; @@ -424,17 +425,26 @@ export class JoyApi { const currentBlock = blockNumber.toBn(); const lockData = await finalizedApi.query.balances.locks.entries(); - const lockDataAdresses: any[] = []; + const systemAccounts = await finalizedApi.query.system.account.entries(); + const vestingLockAddresses: any[] = []; const resultData: { [key: string]: { tempAmount: BN; } & Address; } = {}; - for (let [storageKey, palletBalances] of lockData) { - let biggestLock = new BN(0); - let biggestVestingLock = new BN(0); - const address = storageKey.args[0].toString(); + const updateBaseBalanceData = (address: string, account: FrameSystemAccountInfo) => { + const totalBalance = this.toJOY(account.data.free); + const lockedBalance = this.toJOY( + BN.min(resultData[address].tempAmount, BN.min(account.data.free, account.data.miscFrozen)) + ); + resultData[address].total_balance = totalBalance; + resultData[address].transferrable_balance = totalBalance - lockedBalance; + resultData[address].locked_balance = lockedBalance; + } + + systemAccounts.forEach(([key, account]) => { + const address = key.args[0].toString(); resultData[address] = { tempAmount: new BN(0), @@ -447,6 +457,14 @@ export class JoyApi { vestable: 0, }; + updateBaseBalanceData(address, account); + }); + + for (let [storageKey, palletBalances] of lockData) { + let biggestLock = new BN(0); + let biggestVestingLock = new BN(0); + const address = storageKey.args[0].toString(); + for (let palletBalance of palletBalances) { if(palletBalance.amount.toBn().gt(biggestLock)) { biggestLock = palletBalance.amount.toBn(); @@ -465,16 +483,24 @@ export class JoyApi { } if (biggestVestingLock.gt(new BN(0))) { - lockDataAdresses.push(address); + vestingLockAddresses.push(address); resultData[address].vesting_lock = this.toJOY(biggestVestingLock); } } - const intAccs = await finalizedApi.query.system.account.multi(lockDataAdresses); - const vestingData = await finalizedApi.query.vesting.vesting.multi(lockDataAdresses); + const intAccs = systemAccounts.reduce((acc, [key, account]) => { + const address = key.args[0].toString(); + + if(vestingLockAddresses.includes(address)) { + return [...acc, account]; + } + + return acc; + }, [] as FrameSystemAccountInfo[]); + const vestingData = await finalizedApi.query.vesting.vesting.multi(vestingLockAddresses); intAccs.forEach((val, index) => { - const address = lockDataAdresses[index]; + const address = vestingLockAddresses[index]; const currentAddressVestingData = vestingData[index]; const currentAddressVestingEntries = currentAddressVestingData.unwrapOr(null); @@ -487,18 +513,12 @@ export class JoyApi { return [vestingSumAcc.add(vestingEntry.locked), vestableAcc.add(currentlyVestable)]; }, [new BN(0), new BN(0)]); - const totalBalance = this.toJOY(val.data.free); - const lockedBalance = this.toJOY( - BN.min(resultData[address].tempAmount, BN.min(val.data.free, val.data.miscFrozen)) - ); - resultData[address].total_balance = totalBalance; - resultData[address].transferrable_balance = totalBalance - lockedBalance; - resultData[address].locked_balance = lockedBalance; - if(vestingSum.lte(val.data.free)) { resultData[address].vestable = this.toJOY(vestable); } } + + updateBaseBalanceData(address, val); }); Object.keys(resultData).forEach((address) => { delete (resultData[address] as any).tempAmount; }); From 2916924679f5d37d6a3ea8d956449e0722b0b236 Mon Sep 17 00:00:00 2001 From: Edvin Dzidic Date: Mon, 7 Aug 2023 18:48:54 +0200 Subject: [PATCH 7/7] Simplify address endpoint functionality, update address ui --- public/address_ui.ejs | 1 - src/joyApi.ts | 78 +++++++++++++------------------------------ 2 files changed, 24 insertions(+), 55 deletions(-) diff --git a/public/address_ui.ejs b/public/address_ui.ejs index 0ee560d..61c60ba 100644 --- a/public/address_ui.ejs +++ b/public/address_ui.ejs @@ -25,7 +25,6 @@

Total balance: <%= locals?.total_balance %>

Transferrable balance: <%= locals?.transferrable_balance %>

Vesting lock: <%= locals?.vesting_lock %>

-

Vestable: <%= locals?.vestable %>