From dcf5cdfca0dae4ee7f11476984e1fd096f0e2f1f Mon Sep 17 00:00:00 2001 From: Nitish Sharma Date: Thu, 9 May 2024 15:58:04 +0530 Subject: [PATCH] fixed function in index file --- adapters/block_numbers.tsv | 3 - adapters/zerolend/src/index.ts | 139 +++++++++++++++++++++++---------- adapters/zerolend/yarn.lock | 4 +- 3 files changed, 98 insertions(+), 48 deletions(-) delete mode 100644 adapters/block_numbers.tsv diff --git a/adapters/block_numbers.tsv b/adapters/block_numbers.tsv deleted file mode 100644 index 3b7f0254..00000000 --- a/adapters/block_numbers.tsv +++ /dev/null @@ -1,3 +0,0 @@ -block_timestamp number -1711023841 3041467 -1670496243 0 diff --git a/adapters/zerolend/src/index.ts b/adapters/zerolend/src/index.ts index 542da818..ed58aaa1 100644 --- a/adapters/zerolend/src/index.ts +++ b/adapters/zerolend/src/index.ts @@ -3,65 +3,118 @@ import fs from "fs"; import csv from "csv-parser"; import { BlockData, getUserTVLByBlock } from "./sdk"; +// const readBlocksFromCSV = async (filePath: string): Promise => { +// const blocks: BlockData[] = []; + +// await new Promise((resolve, reject) => { +// fs.createReadStream(filePath) +// .pipe(csv()) // Specify the separator as '\t' for TSV files +// .on("data", (row: any) => { +// const blockNumber = parseInt(row.number, 10); +// const blockTimestamp = parseInt(row.block_timestamp, 10); +// if (!isNaN(blockNumber) && blockTimestamp) { +// blocks.push({ blockNumber: blockNumber, blockTimestamp }); +// } +// }) +// .on("end", resolve) +// .on("error", reject); +// }); + +// return blocks; +// }; + +// readBlocksFromCSV("hourly_blocks.csv") +// .then(async (blocks) => { +// const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks +// const batchSize = 10; // Size of batch to trigger writing to the file +// let i = 0; +// console.log("block number received") +// for (const block of blocks) { +// try { +// const result = await getUserTVLByBlock(block); + +// // Accumulate CSV rows for all blocks +// allCsvRows.push(...result); + +// i++; +// console.log(`Processed block ${i}`); + +// // Write to file when batch size is reached or at the end of loop +// if (i % batchSize === 0 || i === blocks.length) { +// const ws = fs.createWriteStream(`outputData.csv`, { +// flags: i === batchSize ? "w" : "a", +// }); +// write(allCsvRows, { headers: i === batchSize ? true : false }) +// .pipe(ws) +// .on("finish", () => { +// console.log(`CSV file has been written.`); +// }); + +// // Clear the accumulated CSV rows +// allCsvRows.length = 0; +// } +// } catch (error) { +// console.error(`An error occurred for block ${block}:`, error); +// } +// } +// }) +// .catch((err) => { +// console.error("Error reading CSV file:", err); +// }); + +module.exports = { + getUserTVLByBlock, +}; + const readBlocksFromCSV = async (filePath: string): Promise => { const blocks: BlockData[] = []; await new Promise((resolve, reject) => { fs.createReadStream(filePath) - .pipe(csv({ separator: "\t" })) // Specify the separator as '\t' for TSV files - .on("data", (row: any) => { + .pipe(csv()) // Specify the separator as '\t' for TSV files + .on('data', (row) => { const blockNumber = parseInt(row.number, 10); - const blockTimestamp = parseInt(row.block_timestamp, 10); + const blockTimestamp = parseInt(row.timestamp, 10); if (!isNaN(blockNumber) && blockTimestamp) { blocks.push({ blockNumber: blockNumber, blockTimestamp }); } }) - .on("end", resolve) - .on("error", reject); + .on('end', () => { + resolve(); + }) + .on('error', (err) => { + reject(err); + }); }); return blocks; }; -readBlocksFromCSV("block_numbers.tsv") - .then(async (blocks) => { - const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks - const batchSize = 10; // Size of batch to trigger writing to the file - let i = 0; - - for (const block of blocks) { - try { - const result = await getUserTVLByBlock(block); - - // Accumulate CSV rows for all blocks - allCsvRows.push(...result); - - i++; - console.log(`Processed block ${i}`); - - // Write to file when batch size is reached or at the end of loop - if (i % batchSize === 0 || i === blocks.length) { - const ws = fs.createWriteStream(`outputData.csv`, { - flags: i === batchSize ? "w" : "a", - }); - write(allCsvRows, { headers: i === batchSize ? true : false }) - .pipe(ws) - .on("finish", () => { - console.log(`CSV file has been written.`); - }); +readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: BlockData[]) => { + console.log(blocks); + const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks + const batchSize = 1000; // Size of batch to trigger writing to the file + let i = 0; - // Clear the accumulated CSV rows - allCsvRows.length = 0; - } - } catch (error) { - console.error(`An error occurred for block ${block}:`, error); + for (const block of blocks) { + try { + const result = await getUserTVLByBlock(block); + for (let i = 0; i < result.length; i++) { + allCsvRows.push(result[i]) } + } catch (error) { + console.error(`An error occurred for block ${block}:`, error); } - }) - .catch((err) => { - console.error("Error reading CSV file:", err); + } + await new Promise((resolve, reject) => { + const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' }); + write(allCsvRows, { headers: true }) + .pipe(ws) + .on("finish", () => { + console.log(`CSV file has been written.`); + resolve; + }); }); - -module.exports = { - getUserTVLByBlock, -}; +}).catch((err) => { + console.error('Error reading CSV file:', err); +}); \ No newline at end of file diff --git a/adapters/zerolend/yarn.lock b/adapters/zerolend/yarn.lock index f1d62178..1f64dc83 100644 --- a/adapters/zerolend/yarn.lock +++ b/adapters/zerolend/yarn.lock @@ -34,7 +34,7 @@ csv-parser@^3.0.0: version "3.0.0" - resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7" + resolved "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz" integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ== dependencies: minimist "^1.2.0" @@ -89,7 +89,7 @@ lodash.uniq@^4.5.0: minimist@^1.2.0: version "1.2.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== typescript@^5.3.3: