From fb8dbe03334f19166af3b147fbd351e02db9f851 Mon Sep 17 00:00:00 2001 From: Thodoris Greasidis Date: Thu, 19 Sep 2024 09:52:00 +0300 Subject: [PATCH] Stop returning Bluebird promises & drop it from the dependencies Change-type: major --- lib/preload.ts | 342 +++++++++++++++++++++++++++---------------------- package.json | 2 - 2 files changed, 188 insertions(+), 156 deletions(-) diff --git a/lib/preload.ts b/lib/preload.ts index 4a15fd0..672dc21 100644 --- a/lib/preload.ts +++ b/lib/preload.ts @@ -4,7 +4,6 @@ import * as dockerProgress from 'docker-progress'; import * as Docker from 'dockerode'; import * as path from 'path'; import * as streamModule from 'stream'; -import * as Bluebird from 'bluebird'; import * as tarfs from 'tar-fs'; import { promises as fs, constants } from 'fs'; import * as getPort from 'get-port'; @@ -29,6 +28,52 @@ const SPLASH_IMAGE_PATH_IN_DOCKER = '/img/balena-logo.png'; const DOCKER_STEP_RE = /Step (\d+)\/(\d+)/; const CONCURRENT_REQUESTS_TO_REGISTRY = 10; +export const limitedMap = ( + arr: T[], + fn: (currentValue: T, index: number, array: T[]) => Promise, + { + concurrency = CONCURRENT_REQUESTS_TO_REGISTRY, + }: { + concurrency?: number; + } = {}, +): Promise => { + if (concurrency >= arr.length) { + return Promise.all(arr.map(fn)); + } + return new Promise((resolve, reject) => { + const result: U[] = new Array(arr.length); + let inFlight = 0; + let idx = 0; + const runNext = async () => { + // Store the idx to use for this call before incrementing the main counter + const i = idx; + idx++; + if (i >= arr.length) { + return; + } + try { + inFlight++; + result[i] = await fn(arr[i], i, arr); + void runNext(); + } catch (err) { + // Stop any further iterations + idx = arr.length; + // Clear the results so far for gc + result.length = 0; + reject(err); + } finally { + inFlight--; + if (inFlight === 0) { + resolve(result); + } + } + }; + while (inFlight < concurrency) { + void runNext(); + } + }); +}; + const GRAPHDRIVER_ERROR = 'Error starting daemon: error initializing graphdriver: driver not supported'; const OVERLAY_MODULE_MESSAGE = @@ -251,7 +296,7 @@ export class Preloader extends EventEmitter { const build = await this.docker.buildImage(tarStream, { t: DOCKER_IMAGE_TAG, }); - await new Bluebird((resolve, reject) => { + await new Promise((resolve, reject) => { this.docker.modem.followProgress( build, (error, output) => { @@ -337,7 +382,7 @@ export class Preloader extends EventEmitter { * error message including the message provided in the `error` field. */ _runCommand(command: string, parameters: { [name: string]: any }) { - return new Bluebird((resolve, reject) => { + return new Promise((resolve, reject) => { const cmd = JSON.stringify({ command, parameters }) + '\n'; this.stdout.once('error', reject); this.stdout.once('data', (data) => { @@ -675,7 +720,7 @@ export class Preloader extends EventEmitter { } async _getApplicationImagesManifests(imagesLocations, registryToken) { - return await Bluebird.map( + return await limitedMap( imagesLocations, async (imageLocation: string) => { const { body } = await this.registryRequest( @@ -706,15 +751,9 @@ export class Preloader extends EventEmitter { } } } - await Bluebird.map( + await limitedMap( sizeRequests, - async ({ - imageLocation, - layer, - }: { - imageLocation: string; - layer: Layer; - }) => { + async ({ imageLocation, layer }) => { const size = await this._getLayerSize( registryToken, this._registryUrl(imageLocation), @@ -797,25 +836,22 @@ export class Preloader extends EventEmitter { } } - _getRegistryToken(images) { - return Bluebird.resolve( - this.balena.request.send({ - // @ts-expect-error reason unknwon - baseUrl: this.balena.pine.API_URL, - url: '/auth/v1/token', - qs: { - service: this._registryUrl(images[0]), - scope: images.map( - (imageRepository) => - `repository:${imageRepository.substr( - imageRepository.search('/') + 1, - )}:pull`, - ), - }, - }), - ) - .get('body') - .get('token'); + async _getRegistryToken(images: string[]) { + const { body } = await this.balena.request.send({ + // @ts-expect-error reason unknwon + baseUrl: this.balena.pine.API_URL, + url: '/auth/v1/token', + qs: { + service: this._registryUrl(images[0]), + scope: images.map( + (imageRepository) => + `repository:${imageRepository.substr( + imageRepository.search('/') + 1, + )}:pull`, + ), + }, + }); + return body.token as string; } async _fetchApplication() { @@ -894,87 +930,83 @@ export class Preloader extends EventEmitter { return deviceType.is_of__cpu_architecture[0].slug; } - prepare() { - return Bluebird.resolve(this._build()).then(async () => { - // Check that the image is a writable file - await this._runWithSpinner( - 'Checking that the image is a writable file', - () => this._checkImage(this.image), - ); + async prepare() { + await this._build(); + // Check that the image is a writable file + await this._runWithSpinner( + 'Checking that the image is a writable file', + () => this._checkImage(this.image), + ); - // Get a free tcp port and balena sdk settings - const port = await this._runWithSpinner('Finding a free tcp port', () => - getPort(), - ); + // Get a free tcp port and balena sdk settings + const port = await this._runWithSpinner('Finding a free tcp port', () => + getPort(), + ); - this.dockerPort = port; - // Create the docker preloader container - const container = await this._runWithSpinner( - 'Creating preloader container', - () => - createContainer( - this.docker, - this.image, - this.splashImage, - this.dockerPort, - this.proxy, - ), - ); - this.container = container; - await this._runWithSpinner('Starting preloader container', () => - this.container.start(), - ); + this.dockerPort = port; + // Create the docker preloader container + const container = await this._runWithSpinner( + 'Creating preloader container', + () => + createContainer( + this.docker, + this.image, + this.splashImage, + this.dockerPort, + this.proxy, + ), + ); + this.container = container; + await this._runWithSpinner('Starting preloader container', () => + this.container.start(), + ); - await Bluebird.each(this.certificates, (certificate) => { - return this.container.putArchive( - tarfs.pack(path.dirname(certificate), { - entries: [path.basename(certificate)], - }), - { - path: '/usr/local/share/ca-certificates/', - noOverwriteDirNonDir: true, - }, - ); - }); + for (const certificate of this.certificates) { + await this.container.putArchive( + tarfs.pack(path.dirname(certificate), { + entries: [path.basename(certificate)], + }), + { + path: '/usr/local/share/ca-certificates/', + noOverwriteDirNonDir: true, + }, + ); + } - this._prepareErrorHandler(); - const stream = await this.container.attach({ - stream: true, - stdout: true, - stderr: true, - stdin: true, - hijack: true, - }); - this.stdin = stream; - this.docker.modem.demuxStream(stream, this.stdout, this.stderr); - - await Promise.all([ - this._getImageInfo(), - this._fetchDeviceTypes(), - this._fetchApplication(), - ]); + this._prepareErrorHandler(); + const stream = await this.container.attach({ + stream: true, + stdout: true, + stderr: true, + stdin: true, + hijack: true, }); + this.stdin = stream; + this.docker.modem.demuxStream(stream, this.stdout, this.stderr); + + await Promise.all([ + this._getImageInfo(), + this._fetchDeviceTypes(), + this._fetchApplication(), + ]); } - cleanup() { + async cleanup() { // Returns Promise // Deletes the container - return Bluebird.resolve( - this._runWithSpinner('Cleaning up temporary files', async () => { - if (this.container) { - await Promise.all([this.kill(), this.container.wait()]); - await this.container.remove(); - } - }), - ); + await this._runWithSpinner('Cleaning up temporary files', async () => { + if (this.container) { + await Promise.all([this.kill(), this.container.wait()]); + await this.container.remove(); + } + }); } - kill() { + async kill() { // returns Promise if (this.container) { return this.container.kill().catch(() => undefined); } - return Bluebird.resolve(); } _ensureCanPreload() { @@ -1056,62 +1088,64 @@ export class Preloader extends EventEmitter { return '/splash/resin-logo.png'; } - preload() { - return Bluebird.resolve(this._getState()).then(async () => { - this._ensureCanPreload(); - const additionalBytes = await this._runWithSpinner( - 'Estimating required additional space', - () => this._getRequiredAdditionalSpace(), - ); - const images = _.map( - this._getImagesToPreload(), - 'is_stored_at__image_location', - ); - // Wait for dockerd to start - await this._runWithSpinner( - 'Resizing partitions and waiting for dockerd to start', - () => - this._runCommand('preload', { - app_data: this._getAppData(), - additional_bytes: additionalBytes, - splash_image_path: this._getSplashImagePath(), - }), - ); - const registryToken = await this._getRegistryToken(images); - - const opts = { authconfig: { registrytoken: registryToken } }; - // Docker connection - // We use localhost on windows because of this bug in node < 8.10.0: - // https://github.com/nodejs/node/issues/14900 - const innerDocker = new Docker({ - host: os.platform() === 'win32' ? 'localhost' : '0.0.0.0', - port: this.dockerPort, - }); - const innerDockerProgress = new dockerProgress.DockerProgress({ - docker: innerDocker, - }); - const pullingProgressName = `Pulling ${this._pluralize( - images.length, - 'image', - )}`; - // Emit progress events while pulling - const onProgressHandlers = innerDockerProgress.aggregateProgress( - images.length, - (e) => { - this._progress(pullingProgressName, e.percentage); - }, - ); - await Bluebird.map(images, (image, index) => { - return innerDockerProgress.pull(image, onProgressHandlers[index], opts); - }); + async preload() { + await this._getState(); - // Signal that we're done to the Python script. - this.stdin.write('\n'); - // Wait for the script to unmount the data partition - await new Bluebird((resolve, reject) => { - this.stdout.once('error', reject); - this.stdout.once('data', resolve); - }); + this._ensureCanPreload(); + const additionalBytes = await this._runWithSpinner( + 'Estimating required additional space', + () => this._getRequiredAdditionalSpace(), + ); + const images = _.map( + this._getImagesToPreload(), + 'is_stored_at__image_location', + ); + // Wait for dockerd to start + await this._runWithSpinner( + 'Resizing partitions and waiting for dockerd to start', + () => + this._runCommand('preload', { + app_data: this._getAppData(), + additional_bytes: additionalBytes, + splash_image_path: this._getSplashImagePath(), + }), + ); + const registryToken = await this._getRegistryToken(images); + + const opts = { authconfig: { registrytoken: registryToken } }; + // Docker connection + // We use localhost on windows because of this bug in node < 8.10.0: + // https://github.com/nodejs/node/issues/14900 + const innerDocker = new Docker({ + host: os.platform() === 'win32' ? 'localhost' : '0.0.0.0', + port: this.dockerPort, + }); + const innerDockerProgress = new dockerProgress.DockerProgress({ + docker: innerDocker, + }); + const pullingProgressName = `Pulling ${this._pluralize( + images.length, + 'image', + )}`; + // Emit progress events while pulling + const onProgressHandlers = innerDockerProgress.aggregateProgress( + images.length, + (e) => { + this._progress(pullingProgressName, e.percentage); + }, + ); + await Promise.all( + images.map(async (image, index) => { + await innerDockerProgress.pull(image, onProgressHandlers[index], opts); + }), + ); + + // Signal that we're done to the Python script. + this.stdin.write('\n'); + // Wait for the script to unmount the data partition + await new Promise((resolve, reject) => { + this.stdout.once('error', reject); + this.stdout.once('data', resolve); }); } @@ -1125,10 +1159,10 @@ export class Preloader extends EventEmitter { * @param {string} commit * @returns {Promise} */ - setAppIdAndCommit(appIdOrSlug: string | number, commit: string) { + async setAppIdAndCommit(appIdOrSlug: string | number, commit: string) { this.appId = appIdOrSlug; this.commit = commit; this.application = null; - return Bluebird.resolve(this._fetchApplication()); + return await this._fetchApplication(); } } diff --git a/package.json b/package.json index fe63715..210a34c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ ], "dependencies": { "balena-sdk": "^20.1.3", - "bluebird": "^3.7.2", "compare-versions": "^3.6.0", "docker-progress": "^5.0.0", "dockerode": "^4.0.2", @@ -35,7 +34,6 @@ }, "devDependencies": { "@balena/lint": "^7.2.6", - "@types/bluebird": "^3.5.36", "@types/dockerode": "^3.3.23", "@types/node": "^18.0.0", "@types/request-promise": "^4.1.48",