From 90fc2d081e6152decb7b42676933154ff3c05de0 Mon Sep 17 00:00:00 2001 From: tomicvladan Date: Mon, 29 Jan 2024 12:58:51 +0100 Subject: [PATCH 1/5] feat: speed up manifest sync (#490) --- src/command/manifest/sync.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/command/manifest/sync.ts b/src/command/manifest/sync.ts index 43f5e213..e7eb5431 100644 --- a/src/command/manifest/sync.ts +++ b/src/command/manifest/sync.ts @@ -2,11 +2,13 @@ import { readFileSync } from 'fs' import { Argument, LeafCommand, Option } from 'furious-commander' import { Reference } from 'mantaray-js' import { join } from 'path' +import { makeChunkedFile } from '@fairdatasociety/bmt-js' import { pickStamp } from '../../service/stamp' import { readdirDeepAsync } from '../../utils' import { BzzAddress } from '../../utils/bzz-address' import { stampProperties } from '../../utils/option' import { ManifestCommand } from './manifest-command' +import { bytesToHex } from '../../utils/hex' export class Sync extends ManifestCommand implements LeafCommand { public readonly name = 'sync' @@ -48,11 +50,13 @@ export class Sync extends ManifestCommand implements LeafCommand { continue } - const remoteData = await this.bee.downloadData(Buffer.from(fork.node.getEntry).toString('hex')) + const remoteAddress = Buffer.from(fork.node.getEntry).toString('hex') + const localData = readFileSync(join(this.folder, file)) + const rootChunk = makeChunkedFile(localData).rootChunk() + const localAddress = bytesToHex(rootChunk.address()) - // TODO make this more efficient once the chunker is available - if (localData.equals(remoteData)) { + if (localAddress === remoteAddress) { this.console.log('ok -> ' + file) } else { const { reference } = await this.bee.uploadData(this.stamp, readFileSync(join(this.folder, file))) From bb66ea852f2497c4fb0ca4665bcc8dc816b15c97 Mon Sep 17 00:00:00 2001 From: Cafe137 <77121044+Cafe137@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:11:33 +0100 Subject: [PATCH 2/5] chore: optimise manifest sync (#492) --- src/command/feed/print.ts | 4 ++-- src/command/manifest/sync.ts | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/command/feed/print.ts b/src/command/feed/print.ts index 4c305aca..15bda9d3 100644 --- a/src/command/feed/print.ts +++ b/src/command/feed/print.ts @@ -46,14 +46,14 @@ export class Print extends FeedCommand implements LeafCommand { spinner.stop() this.console.verbose(createKeyValue('Chunk Reference', reference)) this.console.verbose(createKeyValue('Chunk Reference URL', `${this.bee.url}/bzz/${reference}/`)) - this.console.verbose(createKeyValue('Feed Index', feedIndex)) + this.console.verbose(createKeyValue('Feed Index', feedIndex as string)) this.console.verbose(createKeyValue('Next Index', feedIndexNext)) this.console.verbose(createKeyValue('Feed Manifest', manifest)) this.console.quiet(manifest) this.console.log(createKeyValue('Topic', `${topic}`)) this.console.log(createKeyValue('Feed Manifest URL', `${this.bee.url}/bzz/${manifest}/`)) - this.console.log(createKeyValue('Number of Updates', parseInt(feedIndex, 16) + 1)) + this.console.log(createKeyValue('Number of Updates', parseInt(feedIndex as string, 16) + 1)) } catch (error) { spinner.stop() const message = getFieldOrNull(error, 'message') diff --git a/src/command/manifest/sync.ts b/src/command/manifest/sync.ts index e7eb5431..ce8241f4 100644 --- a/src/command/manifest/sync.ts +++ b/src/command/manifest/sync.ts @@ -1,14 +1,15 @@ +import { makeChunkedFile } from '@fairdatasociety/bmt-js' +import chalk from 'chalk' import { readFileSync } from 'fs' import { Argument, LeafCommand, Option } from 'furious-commander' import { Reference } from 'mantaray-js' import { join } from 'path' -import { makeChunkedFile } from '@fairdatasociety/bmt-js' import { pickStamp } from '../../service/stamp' import { readdirDeepAsync } from '../../utils' import { BzzAddress } from '../../utils/bzz-address' +import { bytesToHex } from '../../utils/hex' import { stampProperties } from '../../utils/option' import { ManifestCommand } from './manifest-command' -import { bytesToHex } from '../../utils/hex' export class Sync extends ManifestCommand implements LeafCommand { public readonly name = 'sync' @@ -57,23 +58,23 @@ export class Sync extends ManifestCommand implements LeafCommand { const localAddress = bytesToHex(rootChunk.address()) if (localAddress === remoteAddress) { - this.console.log('ok -> ' + file) + this.console.log(chalk.gray(file) + ' ' + chalk.blue('UNCHANGED')) } else { - const { reference } = await this.bee.uploadData(this.stamp, readFileSync(join(this.folder, file))) + const { reference } = await this.bee.uploadData(this.stamp, localData) node.addFork(this.encodePath(file), Buffer.from(reference, 'hex') as Reference) - this.console.log('updated -> ' + file) + this.console.log(chalk.gray(file) + ' ' + chalk.yellow('CHANGED')) } } else { const { reference } = await this.bee.uploadData(this.stamp, readFileSync(join(this.folder, file))) node.addFork(this.encodePath(file), Buffer.from(reference, 'hex') as Reference) - this.console.log('new -> ' + file) + this.console.log(chalk.gray(file) + ' ' + chalk.green('NEW')) } } if (this.remove) { for (const fork of Object.values(forks).filter(x => !x.found)) { node.removePath(this.encodePath(fork.path)) - this.console.log('removed -> ' + fork.path) + this.console.log(chalk.gray(fork.path) + ' ' + chalk.red('REMOVED')) } } await this.saveAndPrintNode(node, this.stamp) From ab3e13e6442a503d38cf6980a3c6257c7accd066 Mon Sep 17 00:00:00 2001 From: Cafe137 <77121044+Cafe137@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:21:55 +0100 Subject: [PATCH 3/5] feat: upgrade to bee-js 6.7.2 (#493) --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 441c51ab..c51b69b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "2.7.0", "license": "BSD-3-Clause", "dependencies": { - "@ethersphere/bee-js": "^6.5.1", + "@ethersphere/bee-js": "^6.7.2", "@fairdatasociety/bmt-js": "^2.1.0", "bignumber.js": "^9.1.0", "chalk": "^2.4.2", @@ -1227,9 +1227,9 @@ } }, "node_modules/@ethersphere/bee-js": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.5.1.tgz", - "integrity": "sha512-qJPdzMAaoFqXzaKlwDaFko68X+FOP2VL9P+yy964boq/hJeDS3d5avo1KtzSR1pIHWs4aPdREvmQ3tgOdOT5+g==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.2.tgz", + "integrity": "sha512-cLwegviwhjXnilOLvvovCLvcgJgxpqTKTQjJ6pmqkR7628ymni3MzJystn70d3Jlai9Xj+Xqju2yyfVtAi0GSg==", "dependencies": { "@ethersphere/swarm-cid": "^0.1.0", "@types/readable-stream": "^2.3.13", @@ -10667,9 +10667,9 @@ } }, "@ethersphere/bee-js": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.5.1.tgz", - "integrity": "sha512-qJPdzMAaoFqXzaKlwDaFko68X+FOP2VL9P+yy964boq/hJeDS3d5avo1KtzSR1pIHWs4aPdREvmQ3tgOdOT5+g==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.2.tgz", + "integrity": "sha512-cLwegviwhjXnilOLvvovCLvcgJgxpqTKTQjJ6pmqkR7628ymni3MzJystn70d3Jlai9Xj+Xqju2yyfVtAi0GSg==", "requires": { "@ethersphere/swarm-cid": "^0.1.0", "@types/readable-stream": "^2.3.13", diff --git a/package.json b/package.json index 0d5ebd76..6aac3f59 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "typescript": "^4.8.4" }, "dependencies": { - "@ethersphere/bee-js": "^6.5.1", + "@ethersphere/bee-js": "^6.7.2", "@fairdatasociety/bmt-js": "^2.1.0", "bignumber.js": "^9.1.0", "chalk": "^2.4.2", From 64b08c831768c2ea3108f61bf0538b4385ca351d Mon Sep 17 00:00:00 2001 From: Cafe137 <77121044+Cafe137@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:27:37 +0100 Subject: [PATCH 4/5] refactor: use bee-js wait for stamp instead of custom (#494) --- package-lock.json | 14 ++++++------- package.json | 2 +- src/command/stamp/buy.ts | 45 ++++------------------------------------ 3 files changed, 12 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index c51b69b6..2baca4b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "2.7.0", "license": "BSD-3-Clause", "dependencies": { - "@ethersphere/bee-js": "^6.7.2", + "@ethersphere/bee-js": "^6.7.3", "@fairdatasociety/bmt-js": "^2.1.0", "bignumber.js": "^9.1.0", "chalk": "^2.4.2", @@ -1227,9 +1227,9 @@ } }, "node_modules/@ethersphere/bee-js": { - "version": "6.7.2", - "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.2.tgz", - "integrity": "sha512-cLwegviwhjXnilOLvvovCLvcgJgxpqTKTQjJ6pmqkR7628ymni3MzJystn70d3Jlai9Xj+Xqju2yyfVtAi0GSg==", + "version": "6.7.3", + "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.3.tgz", + "integrity": "sha512-04AXGXP148DousgjVInyGUlHZnWwTGIgiH5leCACJ6daAUnnJQm7CjQgXIH5y72gGPX1XyzXDlqBlB8zLkihkQ==", "dependencies": { "@ethersphere/swarm-cid": "^0.1.0", "@types/readable-stream": "^2.3.13", @@ -10667,9 +10667,9 @@ } }, "@ethersphere/bee-js": { - "version": "6.7.2", - "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.2.tgz", - "integrity": "sha512-cLwegviwhjXnilOLvvovCLvcgJgxpqTKTQjJ6pmqkR7628ymni3MzJystn70d3Jlai9Xj+Xqju2yyfVtAi0GSg==", + "version": "6.7.3", + "resolved": "https://registry.npmjs.org/@ethersphere/bee-js/-/bee-js-6.7.3.tgz", + "integrity": "sha512-04AXGXP148DousgjVInyGUlHZnWwTGIgiH5leCACJ6daAUnnJQm7CjQgXIH5y72gGPX1XyzXDlqBlB8zLkihkQ==", "requires": { "@ethersphere/swarm-cid": "^0.1.0", "@types/readable-stream": "^2.3.13", diff --git a/package.json b/package.json index 6aac3f59..cc55914f 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "typescript": "^4.8.4" }, "dependencies": { - "@ethersphere/bee-js": "^6.7.2", + "@ethersphere/bee-js": "^6.7.3", "@fairdatasociety/bmt-js": "^2.1.0", "bignumber.js": "^9.1.0", "chalk": "^2.4.2", diff --git a/src/command/stamp/buy.ts b/src/command/stamp/buy.ts index 1be65bd2..7e998236 100644 --- a/src/command/stamp/buy.ts +++ b/src/command/stamp/buy.ts @@ -1,10 +1,9 @@ import { Utils } from '@ethersphere/bee-js' import { LeafCommand, Option } from 'furious-commander' -import { printStamp } from '../../service/stamp' -import { secondsToDhms, sleep } from '../../utils' +import { secondsToDhms } from '../../utils' import { createSpinner } from '../../utils/spinner' import { Storage } from '../../utils/storage' -import { createKeyValue, deletePreviousLine } from '../../utils/text' +import { createKeyValue } from '../../utils/text' import { VerbosityLevel } from '../root-command/command-log' import { StampCommand } from './stamp-command' @@ -87,7 +86,7 @@ export class Buy extends StampCommand implements LeafCommand { return } - const spinner = createSpinner('Buying postage stamp. This may take a few minutes.') + const spinner = createSpinner('Creating postage batch. This may take up to 5 minutes.') if (this.verbosity !== VerbosityLevel.Quiet && !this.curl) { spinner.start() @@ -98,6 +97,7 @@ export class Buy extends StampCommand implements LeafCommand { label: this.label, gasPrice: this.gasPrice?.toString(), immutableFlag: this.immutable, + waitForUsable: this.waitUsable === false ? false : true, }) spinner.stop() this.console.quiet(batchId) @@ -106,42 +106,5 @@ export class Buy extends StampCommand implements LeafCommand { } finally { spinner.stop() } - - if (this.waitUsable) { - await this.waitToBecomeUsable() - } - } - - private async waitToBecomeUsable(): Promise { - const spinner = createSpinner('Waiting for postage stamp to become usable...') - - if (this.verbosity !== VerbosityLevel.Quiet && !this.curl) { - spinner.start() - } - let running = true - - while (running) { - try { - const stamp = await this.beeDebug.getPostageBatch(this.postageBatchId) - - if (!stamp.usable) { - await sleep(1000) - continue - } - - spinner.stop() - - if (this.verbosity === VerbosityLevel.Verbose) { - if (!this.curl) { - deletePreviousLine() - } - printStamp(stamp, this.console, { showTtl: true }) - } - running = false - } catch { - await sleep(1000) - } - } - spinner.stop() } } From bf1c3ea513266b5a369520c02d7477e2671cfe97 Mon Sep 17 00:00:00 2001 From: bee-worker <70210089+bee-worker@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:35:56 +0100 Subject: [PATCH 5/5] chore: release 2.8.0 (#491) --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f714b39a..a9edf0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [2.8.0](https://www.github.com/ethersphere/swarm-cli/compare/v2.7.0...v2.8.0) (2024-02-08) + + +### Features + +* speed up manifest sync ([#490](https://www.github.com/ethersphere/swarm-cli/issues/490)) ([90fc2d0](https://www.github.com/ethersphere/swarm-cli/commit/90fc2d081e6152decb7b42676933154ff3c05de0)) +* upgrade to bee-js 6.7.2 ([#493](https://www.github.com/ethersphere/swarm-cli/issues/493)) ([ab3e13e](https://www.github.com/ethersphere/swarm-cli/commit/ab3e13e6442a503d38cf6980a3c6257c7accd066)) + ## [2.7.0](https://www.github.com/ethersphere/swarm-cli/compare/v2.6.0...v2.7.0) (2023-11-29) diff --git a/package-lock.json b/package-lock.json index 2baca4b0..bb901359 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@ethersphere/swarm-cli", - "version": "2.7.0", + "version": "2.8.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@ethersphere/swarm-cli", - "version": "2.7.0", + "version": "2.8.0", "license": "BSD-3-Clause", "dependencies": { "@ethersphere/bee-js": "^6.7.3", diff --git a/package.json b/package.json index cc55914f..a1d9591a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ethersphere/swarm-cli", - "version": "2.7.0", + "version": "2.8.0", "description": "CLI tool for Bee", "keywords": [ "Bee",