From f1f48a3b7a852e24fd26b3baf6df65b47de6d89e Mon Sep 17 00:00:00 2001 From: James Cramer Date: Sun, 3 Jan 2021 15:54:10 -0500 Subject: [PATCH] 1.0.0 release --- .vscode/launch.json | 2 +- README.md | 79 +---------------------- cache.ts | 6 +- filters.ts | 2 + index.ts | 3 +- interfaces.ts | 3 +- package.json | 5 +- rpc.ts | 44 +++++++------ run-service.sh | 24 +++++++ slpgraphmanager.ts | 152 ++++++++++++++++++++++++++++++++++---------- slptokengraph.ts | 76 ++++------------------ status.ts | 2 +- utxos.ts | 7 ++ 13 files changed, 199 insertions(+), 206 deletions(-) create mode 100755 run-service.sh diff --git a/.vscode/launch.json b/.vscode/launch.json index f8c4d2d7..a9b21167 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,7 +13,7 @@ "sourceMaps": true, "cwd": "${workspaceRoot}", "protocol": "inspector", - "console": "externalTerminal" + "console": "integratedTerminal" }, { "type": "node", diff --git a/README.md b/README.md index fc9d06c5..161a3e1b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ ![SLPDB](assets/slpdb_logo.png) # SLPDB Readme -**Last Updated:** 2020-07-29 +**Last Updated:** 2021-01-03 -**Current SLPDB Version:** 1.0.0-beta-rc12 +**Current SLPDB Version:** 1.0.0 * 1. [What is SLPDB?](#WhatisSLPDB) * 2. [Do you need to install SLPDB?](#DoyouneedtouinstalluSLPDB) @@ -28,7 +28,6 @@ * 7.1 [Parser Tests](#ParserTests) * 7.2 [Input Tests](#InputTests) * 7.3 [Regtest Network Tests](#E2ETests) -* 8. [Change Log](#ChangeLog) @@ -86,7 +85,7 @@ The services `SlpServe` and `SlpSockServer` return query results as a JSON objec ### 4.1. Prerequisites * Node.js 12 -* MongoDB 4.0+ +* MongoDB 4.4+ * BitcoinABC, Bitcoin Cash Node, BitcoinUnlimited, BCHD, or other Bitcoin Cash full node with: * RPC-JSON (or gRPC) and * ZeroMQ event notifications @@ -348,75 +347,3 @@ The SLPJS npm library also passes all unit tests which test for the specified in ### 7.3. End-to-End Tests A set of end-to-end tests have been created in order to ensure the expected behavior of SLPDB utilizing the bitcoin regtest network. These tests simulate actual transaction activity using the bitcoin regtest test network and check for proper state in mongoDB and also check that zmq notifications are emitted. The `tests` directory contains the end-to-end tests which can be run by following the instructions provided in the `regtest` directory. - - - -## 8. Change Log - -* 1.0.0 - * Removed `utxos` and `addresses` collections, as this information can be queried from the `graphs` collection. Updated README examples for queries which relied on now defunct `utxos` and `addresses` collections - * Added Topological sorting in block crawl to allow for token validation during block crawl - * Fixed issue where unconfirmed/confirmed transactions where initially writen to db without SLP property and then updated later after a subsequent validation step. - * Fixed issue where MINT baton address is wrong if not the same as token receiver address. - * Added graph pruning to actively reduce memory footprint - * Move mint baton status (mintBatonStatus) and txo out of statistics (mintBatonUtxo), these are now main property of the token document - * Update notification format for consistent value type, always returning string based numbers, no more Decimal128 in block notification - * Removed token stats properties: `block_last_active_send`, `block_last_active_mint`, `qty_valid_token_utxos`, `qty_valid_token_addresses` - * Removed token stats properties: `qty_token_minted`, `qty_token_burned`, `qty_token_circulating_supply` - * Removed UTXO status enums: `SPENT_INVALID_SLP`, `BATON_SPENT_INVALID_SLP` - * Renamed `qty_valid_txns_since_genesis` to `approx_txns_since_genesis`, since this can be corrupted if the block checkpoint is reset. - * Removed network dependency to rest.bitcoin.com - * Add env var to skip the initial full node sync check that happens only on startup - -* 0.15.6 - * Bug fixes and improvements in slpjs - -* 0.15.5 - * Various bug fixes and improvements - * NOTE: this version was not published - -* 0.15.4 - * Added statuses collection - * Added env variables for disabling graph search and zmq publishing - * Caching improvements to speed up token processing - * Various fixes and improvements - -* 0.15.3 - * Additional db cleanup needed in graphs collection from the issue fixed in 0.15.2 - * Removed unneeded object initializers in FromDbObjects, moved to TokenGraph constructor - -* 0.15.2 - * Fixed issue with address format occurring when restarting SLPDB - * Cleaned up readme file formatting (Jt) - -* 0.15.1 - * Fix issue with minting baton status update on initiating token from scratch - -* 0.15.0 - * Add Graph Search metadata (see example queries above for search examples) - * Start listening to blockchain updates before the startup process for improved data integrity during long startup periods - * Fixed bug where not all inputs were showing up in graph transactions - * Replaced getinfo RPC in favor of getBlockchainInfo - * Fixed issue with tokens collection items missing "block_created" data - -* 0.14.1 - * Improved token burn detection after a new blocks is broadcast - * Updated to slpjs version 0.21.1 with caching bug fix. - * Updated updateTxnCollections() so that any valid transactions erroneously marked as "invalid" get updated. - * Refactored several methods to use destructured method parameters - * Other minor refactors and improvements - -* 0.14.0 - * Added NFT1 support - * Improved token burn detection on new block event - * Prevent duplicate zmq transaction publish notifications - * Various bug fixes and improvements - -* 0.13.0 - * Breaking Change: This change may impact any application using `TokenUtxoStatus` or the Graph collection's `graphTxn.outputs.status` property. A new enum type was added to `TokenUtxoStatus` called "EXCESS_INPUT_BURNED". This status is applied to any transaction that has an input SLP quantity that is greater than output SLP quantity. This enum label is used within the Graphs collection's `graphTxn.outputs.status` property. - * A new startup script called "reprocess" was added for the purpose of debugging. Usage is `node index.js reprocess `. - * Fixed issue with block and mempool item cache used to ignore duplicate zmq notifications - * Fixed testnet config starting block (off by one) - * P2MS and P2PK output addresses are stored as `scriptPubKey:` - * Reduced number of RPC calls by subscribing to zmq 'rawtx' instead of 'hashtx' - * Other minor improvements diff --git a/cache.ts b/cache.ts index 7885557d..83da57db 100644 --- a/cache.ts +++ b/cache.ts @@ -111,8 +111,9 @@ export class CacheMap { } delete(key: T) { - if(this.map.delete(key)) + if (this.map.delete(key)) { this.list = this.list.filter(k => k !== key); + } } toMap() { @@ -121,8 +122,9 @@ export class CacheMap { private shift(): T | undefined { let key = this.list.shift(); - if(key) + if(key) { this.map.delete(key); + } return key; } diff --git a/filters.ts b/filters.ts index 1a5d96a9..87af6323 100644 --- a/filters.ts +++ b/filters.ts @@ -94,3 +94,5 @@ class _TokenFilter { // accessor to a singleton stack for filters export const TokenFilters = _TokenFilter.Instance; + +TokenFilters(); \ No newline at end of file diff --git a/index.ts b/index.ts index f0ded65a..7ecf1bdd 100644 --- a/index.ts +++ b/index.ts @@ -30,7 +30,6 @@ let db = new Db({ dbUrl: Config.db.url, config: Config.db }); -let filter = TokenFilters(); let bit = new Bit(db); new SlpdbStatus(db, process.argv); @@ -82,7 +81,7 @@ const daemon = { let currentHeight = await RpcClient.getBlockCount(); tokenManager = new SlpGraphManager(db, currentHeight, network, bit); bit._slpGraphManager = tokenManager; - let pruningStack = PruneStack(tokenManager._tokens); + PruneStack(tokenManager._tokens); // call instantiates singleton console.log('[INFO] Synchronizing SLPDB with BCH blockchain data...', new Date()); console.time('[PERF] Initial Block Sync'); diff --git a/interfaces.ts b/interfaces.ts index d0c19739..3e1c21c2 100644 --- a/interfaces.ts +++ b/interfaces.ts @@ -17,7 +17,6 @@ export interface GraphTxnOutput { vout: number; bchSatoshis: number|null; slpAmount: BigNumber; - isBurnCounted?: boolean; spendTxid: string | null; status: TokenUtxoStatus|BatonUtxoStatus; invalidReason: string | null; @@ -26,7 +25,7 @@ export interface GraphTxnOutput { export interface GraphTxnInput { txid: string; vout: number; - slpAmount: BigNumber; + slpAmount: BigNumber; address: string; bchSatoshis: number; } diff --git a/package.json b/package.json index 4e5ae786..7fb4f53e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "slpdb", - "version": "1.0.0-beta-rc13", + "version": "1.0.0", "description": "Indexer for the Simple Ledger Protocol with real-time validation notifications.", "main": "index.js", "scripts": { @@ -16,7 +16,6 @@ "keywords": [ "SLP" ], - "author": "James Cramer", "license": "MIT", "bugs": { "url": "https://github.com/simpleledger/SLPDB/issues" @@ -47,7 +46,7 @@ "level": "^5.0.1", "migrate-mongo": "^5.0.1", "mingo": "^2.5.3", - "mongodb": "^3.5.11", + "mongodb": "^3.5.6", "node-jq": "1.6.0", "os-utils": "0.0.14", "p-limit": "2.0.0", diff --git a/rpc.ts b/rpc.ts index 70d98274..ae29ad0a 100644 --- a/rpc.ts +++ b/rpc.ts @@ -14,12 +14,11 @@ export class RpcClient { static transactionCache = new CacheMap(500000); static useGrpc: boolean | undefined; constructor({ useGrpc }: { useGrpc?: boolean }) { - if(useGrpc) { + if (useGrpc) { RpcClient.useGrpc = useGrpc; - if(Boolean(Config.grpc.url) && Config.grpc.certPath) { + if (Boolean(Config.grpc.url) && Config.grpc.certPath) { grpc = new GrpcClient({ url: Config.grpc.url, rootCertPath: Config.grpc.certPath }); - } - else { + } else { grpc = new GrpcClient({ url: Config.grpc.url }); } } else { @@ -33,7 +32,7 @@ export class RpcClient { } static async getBlockCount(): Promise { - if(this.useGrpc) { + if (this.useGrpc) { console.log("[INFO] gRPC: getBlockchainInfo"); return (await grpc.getBlockchainInfo()).getBestHeight(); } @@ -42,7 +41,7 @@ export class RpcClient { } static async getBlockchainInfo(): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getBlockchainInfo"); let info = await grpc.getBlockchainInfo(); return { @@ -64,7 +63,7 @@ export class RpcClient { } static async getBlockHash(block_index: number, asBuffer=false): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getBlockInfo (for getBlockHash)"); let hash = Buffer.from((await grpc.getBlockInfo({ index: block_index })).getInfo()!.getHash_asU8().reverse()); if(asBuffer) { @@ -81,7 +80,7 @@ export class RpcClient { } static async getRawBlock(hash: string): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getRawBlock"); return Buffer.from((await grpc.getRawBlock({ hash: hash, reversedHashOrder: true })).getBlock_asU8()).toString('hex') } @@ -89,13 +88,15 @@ export class RpcClient { } static async getBlockInfo({ hash, index }: { hash?: string, index?: number}): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getBlockInfo"); let blockinfo: BlockInfo; - if(index) + if (index) { blockinfo = (await grpc.getBlockInfo({ index: index })).getInfo()!; - else + } else { blockinfo = (await grpc.getBlockInfo({ hash: hash, reversedHashOrder: true })).getInfo()!; + } + return { hash: Buffer.from(blockinfo.getHash_asU8().reverse()).toString('hex'), confirmations: blockinfo.getConfirmations(), @@ -117,8 +118,7 @@ export class RpcClient { if (index) { console.log("[INFO] JSON RPC: getBlockInfo/getBlockHash", index); hash = await rpc.getBlockHash(index); - } - else if (!hash) { + } else if (!hash) { throw Error("No index or hash provided for block") } @@ -127,7 +127,7 @@ export class RpcClient { } static async getRawMemPool(): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getRawMemPool"); return (await grpc.getRawMempool({ fullTransactions: false })).getTransactionDataList().map(t => Buffer.from(t.getTransactionHash_asU8().reverse()).toString('hex')) } @@ -136,16 +136,18 @@ export class RpcClient { } static async getRawTransaction(hash: string, retryRpc=true): Promise { - if(RpcClient.transactionCache.has(hash)) { + if (RpcClient.transactionCache.has(hash)) { console.log("[INFO] cache: getRawTransaction"); return RpcClient.transactionCache.get(hash)!.toString('hex'); } - if(RpcClient.useGrpc) { + + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getRawTransaction", hash); return Buffer.from((await grpc.getRawTransaction({ hash: hash, reversedHashOrder: true })).getTransaction_asU8()).toString('hex'); - } + } + console.log("[INFO] JSON RPC: getRawTransaction", hash); - if(retryRpc) { + if (retryRpc) { return await rpc_retry.getRawTransaction(hash); } else { return await rpc.getRawTransaction(hash); @@ -153,7 +155,7 @@ export class RpcClient { } static async getTransactionBlockHash(hash: string): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { console.log("[INFO] gRPC: getTransaction", hash); let txn = await grpc.getTransaction({ hash: hash, reversedHashOrder: true }); return Buffer.from(txn.getTransaction()!.getBlockHash_asU8().reverse()).toString('hex'); @@ -163,7 +165,7 @@ export class RpcClient { } static async getTxOut(hash: string, vout: number): Promise { - if(RpcClient.useGrpc){ + if (RpcClient.useGrpc){ console.log("[INFO] gRPC: getTxOut", hash, vout); try { let utxo = (await grpc.getUnspentOutput({ hash: hash, vout: vout, reversedHashOrder: true, includeMempool: true })); @@ -177,7 +179,7 @@ export class RpcClient { } static async getMempoolInfo(): Promise { - if(RpcClient.useGrpc) { + if (RpcClient.useGrpc) { return {}; } console.log("[INFO] JSON RPC: getMempoolInfo"); diff --git a/run-service.sh b/run-service.sh new file mode 100755 index 00000000..56bbf214 --- /dev/null +++ b/run-service.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +cd /root/SLPDB/ + +echo "Compiling..." +./node_modules/typescript/bin/tsc +echo "Compiling done." + +echo "Checking for DB migrations..." +export db_url=mongodb://127.0.0.1:27017 +./node_modules/migrate-mongo/bin/migrate-mongo.js up +echo "Finished DB migrations." + +FLAG=./ctl/REPROCESS +if [ -f "$FLAG" ]; then + echo "Found REPROCESS file flag" + echo "node --max_old_space_size=8192 ./index.js run --reprocess" + node --max_old_space_size=8192 ./index.js run --reprocess +else + echo "Starting normally based on CMD" + echo "node --max_old_space_size=8192 ./index.js $@" + node --max_old_space_size=8192 ./index.js "$@" +fi + diff --git a/slpgraphmanager.ts b/slpgraphmanager.ts index 2076ce20..caed1a6c 100644 --- a/slpgraphmanager.ts +++ b/slpgraphmanager.ts @@ -48,31 +48,104 @@ export class SlpGraphManager { } } - async getTokenGraph({ tokenIdHex, slpMsgDetailsGenesis, forceValid, blockCreated, nft1ChildParentIdHex, txid }: { tokenIdHex: string, slpMsgDetailsGenesis?: SlpTransactionDetails, forceValid?: boolean, blockCreated?: number, nft1ChildParentIdHex?: string, txid: string }): Promise { + async getTokenGraph({ tokenIdHex, slpMsgDetailsGenesis, forceValid, blockCreated, nft1ChildParentIdHex, txid }: { tokenIdHex: string, slpMsgDetailsGenesis?: SlpTransactionDetails, forceValid?: boolean, blockCreated?: number, nft1ChildParentIdHex?: string, txid?: string }): Promise { + let filter = TokenFilters(); if (!filter.passesAllFilterRules(tokenIdHex)) { throw Error("Token is filtered and will not be processed, even though it's graph may be loaded.") } - if (!this._tokens.has(tokenIdHex)) { + + if (! this._tokens.has(tokenIdHex)) { + if (!slpMsgDetailsGenesis) { throw Error(`Token details for a new token GENESIS must be provided (Id: ${tokenIdHex}, txid: ${txid}).`); } + if (slpMsgDetailsGenesis.transactionType !== SlpTransactionType.GENESIS) { throw Error(`Missing token details for a non-GENESIS transaction (Id: ${tokenIdHex}, txid: ${txid}).`); } - let graph = new SlpTokenGraph(slpMsgDetailsGenesis, this.db, this, this._network, blockCreated!); + + let tg = new SlpTokenGraph(slpMsgDetailsGenesis, this, blockCreated!, null); + tg._loadInitiated = true; + + if (nft1ChildParentIdHex) { + tg._nftParentId = nft1ChildParentIdHex; + } + + if (!tg._nftParentId && slpMsgDetailsGenesis.versionType === SlpVersionType.TokenVersionType1_NFT_Child) { + await tg.setNftParentId(); + } + + // If NFT child, then we need make sure the child's validation cache object is from the parent graph + if (tg._nftParentId) { + let ptg = await this.getTokenGraph({ tokenIdHex: tg._nftParentId }); + if (! ptg) { + throw Error("this should never happen"); + } + tg._slpValidator.cachedValidations = ptg._slpValidator.cachedValidations; + } + if (forceValid) { - graph._isValid = true; - } else if (!(await graph.IsValid())) { + tg._isValid = true; + } else if (!(await tg.IsValid())) { return null; } - if (nft1ChildParentIdHex) { - graph._nftParentId; + this._tokens.set(tokenIdHex, tg); + + } else if (! this._tokens.get(tokenIdHex)!._lazilyLoaded && ! this._tokens.get(tokenIdHex)!._loadInitiated) { + + let tg = this._tokens.get(tokenIdHex!); + + let lastPrunedHeight = tg!._tokenDbo!._pruningState.pruneHeight; + let checkpoint = await Info.getBlockCheckpoint(); + if (lastPrunedHeight > checkpoint.height) { + lastPrunedHeight = checkpoint.height; + } + + console.log(`[INFO] Lazily loading token ${tokenIdHex}`); + let unspentDag: GraphTxnDbo[] = await this.db.graphFetch(tokenIdHex, lastPrunedHeight); + if (! tg) { + throw Error("This should never happen"); + } + + // set loading state + tg._loadInitiated = true; + tg._lazilyLoaded = true; + + // add minting baton + tg._mintBatonUtxo = tg._tokenDbo!.mintBatonUtxo; + tg._mintBatonStatus = tg._tokenDbo!.mintBatonStatus; + + // Map _txnGraph + tg._graphTxns.fromDbos( + unspentDag, + tg._tokenDbo!._pruningState + ); + + // If NFT child, then we need make sure the child's validation cache object is from the parent graph + if (tg._nftParentId) { + let ptg = await this.getTokenGraph({ tokenIdHex: tg._nftParentId }); + if (! ptg) { + throw Error("this should never happen"); + } + tg._slpValidator.cachedValidations = ptg._slpValidator.cachedValidations; } - if (!graph._nftParentId && slpMsgDetailsGenesis.versionType === SlpVersionType.TokenVersionType1_NFT_Child) { - await graph.setNftParentId(); + + // preload with cachedValidations for this tokenID + for (let [txid, _] of tg._graphTxns) { + let validation: any = { validity: null, details: null, invalidReason: null, parents: [], waiting: false } + validation.validity = tg._graphTxns.get(txid) ? true : false; + validation.details = tg._graphTxns.get(txid)!.details; + if (!validation.details) { + throw Error("No saved details about transaction" + txid); + } + tg._slpValidator.cachedValidations[txid] = validation; } - this._tokens.set(tokenIdHex, graph); + + console.log(`[INFO] Loaded ${tg._graphTxns.size} validation cache results`); + + // Map _lastUpdatedBlock + tg._lastUpdatedBlock = tg._tokenDbo!.lastUpdatedBlock; } else if (slpMsgDetailsGenesis && blockCreated && !this._tokens.get(tokenIdHex)!._blockCreated) { this._tokens.get(tokenIdHex)!._blockCreated = blockCreated; } @@ -201,13 +274,12 @@ export class SlpGraphManager { this._network = network; this._tokens = new Map(); this._bit = bit; - let self = this; this._startupTokenCount = 0 } static MapTokenDetailsToTnaDbo(details: SlpTransactionDetails, genesisDetails: SlpTransactionDetails, addresses: (string|null)[]): SlpTransactionDetailsTnaDbo { var outputs: any|null = null; - if(details.sendOutputs) { + if (details.sendOutputs) { outputs = []; details.sendOutputs.forEach((o,i) => { if (i > 0) { @@ -215,7 +287,7 @@ export class SlpGraphManager { } }) } - if(details.genesisOrMintQuantity) { + if (details.genesisOrMintQuantity) { outputs = []; outputs.push({ address: addresses[0], amount: Decimal128.fromString(details.genesisOrMintQuantity!.dividedBy(10**genesisDetails.decimals).toFixed()) }) } @@ -236,33 +308,43 @@ export class SlpGraphManager { } async initAllTokenGraphs() { - let tokens = await this.db.tokenFetchAll(); - let checkpoint = await Info.getBlockCheckpoint(); - if (tokens) { + let tokenDbos = await this.db.tokenFetchAll(); + if (tokenDbos) { let count = 0; - for (let token of tokens) { - if (token.schema_version !== Config.db.token_schema_version) { + for (let tokenDbo of tokenDbos) { + + if (tokenDbo.schema_version !== Config.db.token_schema_version) { throw Error("DB schema does not match the current version."); } - let lastPrunedHeight = token._pruningState.pruneHeight; - if (lastPrunedHeight > checkpoint.height) { - lastPrunedHeight = checkpoint.height; + + let tokenIdHex = tokenDbo.tokenDetails.tokenIdHex; + + let filter = TokenFilters(); + if (!filter.passesAllFilterRules(tokenIdHex)) { + throw Error("Token is filtered and will not be processed, even though it's graph may be loaded.") } - await this.loadTokenFromDb(token, lastPrunedHeight); - console.log(`[INFO] ${++count} tokens loaded from db.`) + + let tokenDetails = SlpTokenGraph.MapDbTokenDetailsFromDbo(tokenDbo.tokenDetails, tokenDbo.tokenDetails.decimals); + if (! tokenDetails) { + throw Error(`Token details for a new token GENESIS must be provided (Id: ${tokenIdHex}).`); + } + + if (tokenDetails.transactionType !== SlpTransactionType.GENESIS) { + throw Error(`Missing token details for a non-GENESIS transaction (Id: ${tokenIdHex}).`); + } + + let graph = new SlpTokenGraph(tokenDetails, this, tokenDbo.tokenStats.block_created!, tokenDbo); + if (tokenDetails.versionType === SlpVersionType.TokenVersionType1_NFT_Child) { + graph._nftParentId = tokenDbo.nftParentId; + } + this._tokens.set(tokenDbo.tokenDetails.tokenIdHex, graph); + + console.log(`[INFO] Loaded ${tokenIdHex}.`); + count++; } - } - } - async loadTokenFromDb(tokenDbo: TokenDBObject, lastPrunedHeight?: number) { - let tokenId = tokenDbo.tokenDetails.tokenIdHex; - console.log("########################################################################################################"); - console.log(`LOAD FROM DB: ${tokenId}`); - console.log("########################################################################################################"); - let unspentDag: GraphTxnDbo[] = await this.db.graphFetch(tokenId, lastPrunedHeight); - this._cacheGraphTxnCount += unspentDag.length; - console.log(`Total loaded: ${this._cacheGraphTxnCount}, using a pruning cutoff height of: ${lastPrunedHeight} `); - return await SlpTokenGraph.initFromDbos(tokenDbo, unspentDag, this, this._network); + console.log(`[INFO] ${count} tokens loaded from db.`); + } } async stop() { @@ -274,7 +356,7 @@ export class SlpGraphManager { } let unspentCount = 0; - for (let [tokenId, token] of this._tokens) { + for (let [_, token] of this._tokens) { await token.stop(); unspentCount += token.graphSize; } diff --git a/slptokengraph.ts b/slptokengraph.ts index 6cd00301..8cfd1639 100644 --- a/slptokengraph.ts +++ b/slptokengraph.ts @@ -10,8 +10,8 @@ import * as pQueue from 'p-queue'; import { DefaultAddOptions } from 'p-queue'; import { SlpGraphManager } from './slpgraphmanager'; import { CacheMap } from './cache'; -import { TokenDBObject, GraphTxnDbo, SlpTransactionDetailsDbo, TokenUtxoStatus, - BatonUtxoStatus, TokenBatonStatus, GraphTxn } from './interfaces'; +import { SlpTransactionDetailsDbo, TokenUtxoStatus, + BatonUtxoStatus, TokenBatonStatus, GraphTxn, TokenDBObject } from './interfaces'; import { GraphMap } from './graphmap'; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -34,7 +34,7 @@ export class SlpTokenGraph { _mintBatonUtxo = ""; _mintBatonStatus = TokenBatonStatus.UNKNOWN; _nftParentId?: string; - private _graphTxns: GraphMap; + _graphTxns: GraphMap; _slpValidator = new LocalValidator(bitbox, async (txids) => { // if (this._manager._bit.doubleSpendCache.has(txids[0])) { // return [ Buffer.alloc(60).toString('hex') ]; @@ -56,18 +56,21 @@ export class SlpTokenGraph { _manager: SlpGraphManager; _startupTxoSendCache?: CacheMap; _loadInitiated = false; + _lazilyLoaded = false; _updateComplete = true; _isValid?: boolean; + _tokenDbo: TokenDBObject|null; - constructor(tokenDetails: SlpTransactionDetails, db: Db, manager: SlpGraphManager, network: string, blockCreated: number|null) { + constructor(tokenDetails: SlpTransactionDetails, manager: SlpGraphManager, blockCreated: number|null, tokenDbo: TokenDBObject|null) { this._tokenDetails = tokenDetails; this._tokenIdHex = tokenDetails.tokenIdHex; this._tokenIdBuf = Buffer.from(this._tokenIdHex, "hex"); this._graphTxns = new GraphMap(this); - this._db = db; + this._db = manager.db; this._manager = manager; - this._network = network; + this._network = manager._network; this._blockCreated = blockCreated; + this._tokenDbo = tokenDbo; } get graphSize() { @@ -144,7 +147,7 @@ export class SlpTokenGraph { public async validateTxid(txid: string) { await this._slpValidator.isValidSlpTxid(txid, this._tokenIdHex); const validation = this._slpValidator.cachedValidations[txid]; - if (!validation.validity) { + if (! validation.validity) { delete this._slpValidator.cachedValidations[txid]; delete this._slpValidator.cachedRawTransactions[txid]; } @@ -183,10 +186,10 @@ export class SlpTokenGraph { let nftBurnTxnHex = await RpcClient.getRawTransaction(tx.inputs[0].previousTxHash); let nftBurnTxn = Primatives.Transaction.parseFromBuffer(Buffer.from(nftBurnTxnHex, 'hex')); let nftBurnSlp = slp.parseSlpOutputScript(Buffer.from(nftBurnTxn.outputs[0].scriptPubKey)); + if (nftBurnSlp.transactionType === SlpTransactionType.GENESIS) { this._nftParentId = tx.inputs[0].previousTxHash; - } - else { + } else { this._nftParentId = nftBurnSlp.tokenIdHex; } } @@ -331,7 +334,7 @@ export class SlpTokenGraph { public async queueAddGraphTransaction({ txid }: { txid: string }): Promise { let self = this; - while (this._loadInitiated && !this.IsLoaded) { + while (this._loadInitiated && !this.IsLoaded && this._tokenIdHex !== txid) { console.log(`Waiting for token ${this._tokenIdHex} to finish loading...`); await sleep(250); } @@ -646,59 +649,6 @@ export class SlpTokenGraph { return res; } - - static async initFromDbos(token: TokenDBObject, dag: GraphTxnDbo[], manager: SlpGraphManager, network: string): Promise { - let tokenDetails = this.MapDbTokenDetailsFromDbo(token.tokenDetails, token.tokenDetails.decimals); - // if (!token.tokenStats.block_created && token.tokenStats.block_created !== 0) { - // throw Error("Must have a block created for token"); - // } - let tg = await manager.getTokenGraph({ - txid: token.tokenDetails.tokenIdHex, - tokenIdHex: token.tokenDetails.tokenIdHex, - slpMsgDetailsGenesis: tokenDetails, - forceValid: true, - blockCreated: token.tokenStats.block_created!, - nft1ChildParentIdHex: token.nftParentId - }); - if (!tg) { - throw Error("This should never happen"); - } - tg._loadInitiated = true; - - // add minting baton - tg._mintBatonUtxo = token.mintBatonUtxo; - tg._mintBatonStatus = token.mintBatonStatus; - - // add nft parent id - if (token.nftParentId) { - tg._nftParentId = token.nftParentId; - } - - tg._network = network; - - // Map _txnGraph - tg!._graphTxns.fromDbos( - dag, - token._pruningState - ); - - // Preload SlpValidator with cachedValidations - tg._graphTxns.forEach((_, txid) => { - let validation: any = { validity: null, details: null, invalidReason: null, parents: [], waiting: false } - validation.validity = tg!._graphTxns.get(txid) ? true : false; - validation.details = tg!._graphTxns.get(txid)!.details; - if(!validation.details) - throw Error("No saved details about transaction" + txid); - tg!._slpValidator.cachedValidations[txid] = validation; - }); - - console.log(`[INFO] Loaded ${tg._graphTxns.size} validation cache results`); - - // Map _lastUpdatedBlock - tg._lastUpdatedBlock = token.lastUpdatedBlock; - - return tg; - } } // export interface AddressBalance { diff --git a/status.ts b/status.ts index 01ebe73a..c16936dc 100644 --- a/status.ts +++ b/status.ts @@ -191,7 +191,7 @@ export class SlpdbStatus { console.log(`[INFO] Telementry response code: ${res.statusCode}`); res.on('data', d => { console.log(`[INFO] Telemetry response from ${Config.telemetry.host}: ${d.toString('utf8')}`); - Info.setTelemetrySecret(d.secretKey); + JSON.parse(d).secretKey ? Info.setTelemetrySecret(JSON.parse(d).secretKey) : null; }); }); req.on('error', error => { diff --git a/utxos.ts b/utxos.ts index 88bfec30..3b26a79a 100644 --- a/utxos.ts +++ b/utxos.ts @@ -6,6 +6,13 @@ class GlobalUtxoSet extends Map { return this._instance || (this._instance = new GlobalUtxoSet()); } private static _instance: GlobalUtxoSet; + + public set(key: string, value: Buffer): this { + if (this.size % 100000 === 0) { + console.log(`UTXO size: ${this.size}`); + } + return super.set(key, value); + } private constructor() { super(); } }