diff --git a/svc/backfill.ts b/svc/backfill.ts index 156f1bdc7..cd3d3fd41 100644 --- a/svc/backfill.ts +++ b/svc/backfill.ts @@ -12,17 +12,18 @@ import fs from 'fs'; // BLOB_ARCHIVE_S3_BUCKET: 'opendota-blobs', // This endpoint is limited to something like 1 request every 5 seconds -const SCANNER_WAIT = 500; +const SCANNER_WAIT = 1000; const blobArchive = new Archive('blob'); // We can stop at approximately 6400000000 (Feb 2024) const stop = Number(process.env.BACKFILL_STOP) || 6400000000; -async function scanApi() { +async function scanApi(seqNum: number) { + let nextSeqNum = seqNum; while (true) { - let nextSeqNum = Number(fs.readFileSync('./match_seq_num.txt')) || 0; if (nextSeqNum > stop) { process.exit(0); } + const begin = Date.now(); const container = generateJob('api_sequence', { start_at_match_seq_num: nextSeqNum, }); @@ -51,34 +52,39 @@ async function scanApi() { console.log('[API] match_seq_num:%s, matches:%s', nextSeqNum, resp.length); // write to blobstore, process the blob using same function as insertMatch try { - const insertResult = await Promise.all(resp.map(async (origMatch: ApiMatch) => { - const match = transformMatch(origMatch); - // ifNotExists functionality not implemented by some s3 providers - const result = await blobArchive.archivePut(match.match_id + '_api', Buffer.from(JSON.stringify(match)), false); - if (!result) { - throw new Error('failed to insert match ' + match.match_id); + const insertResult = await Promise.all(resp.map(async (origMatch: ApiMatch) => { + const match = transformMatch(origMatch); + // ifNotExists functionality not implemented by some s3 providers + const result = await blobArchive.archivePut(match.match_id + '_api', Buffer.from(JSON.stringify(match)), false); + if (!result) { + throw new Error('failed to insert match ' + match.match_id); + } + })); + if (resp.length) { + nextSeqNum = resp[resp.length - 1].match_seq_num + 1; + console.log('next_seq_num: %s', nextSeqNum); + // Completed inserting matches on this page so update + fs.writeFileSync('./match_seq_num.txt', nextSeqNum.toString()); } - })); - if (resp.length) { - nextSeqNum = resp[resp.length - 1].match_seq_num + 1; - console.log('next_seq_num: %s', nextSeqNum); - // Completed inserting matches on this page so update - fs.writeFileSync('./match_seq_num.txt', nextSeqNum.toString()); + } catch (e) { + // If any fail, log the error and try the same number again + console.error(e); } - } catch (e) { - // If any fail, log the error and try the same number again - console.error(e); - } + const end = Date.now(); + const elapsed = end - begin; + const adjustedWait = Math.max(SCANNER_WAIT - elapsed, 0); + console.log('iteration: %dms', elapsed); await new Promise((resolve) => setTimeout( resolve, - SCANNER_WAIT, + adjustedWait, ), ); } } async function start() { - await scanApi(); + let seqNum = Number(fs.readFileSync('./match_seq_num.txt')) || 0; + await scanApi(seqNum); } start();