diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f2611c8..abb9306 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.GH_TOKEN }} + token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} - uses: actions/cache@v3 with: path: ./node_modules @@ -44,7 +44,7 @@ jobs: eslint_extensions: ts prettier: true prettier_extensions: ts,json,js,yml - auto_fix: true + auto_fix: ${{ secrets.GH_TOKEN && true || false }} commit_message: 'chore(lint): Fix code style issues with ${linter}' github_token: ${{ secrets.GITHUB_TOKEN }} git_email: 'lint-action@github.com' diff --git a/README.md b/README.md index c60bcea..033fb2b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The following environment variables are supported: | `REDIS_PORT` | x | `null` | Redis port | | `REDIS_PASSWORD` | | `null` | Redis password | | `REDIS_DB` | | `0` | Redis database index to use (see `options.db` from [docs](https://ioredis.readthedocs.io/en/latest/API/#new-redisport-host-options)) | +| `REDIS_FAMILY` | | `` | Redis connection family to use (see `options.family` from [docs](https://ioredis.readthedocs.io/en/latest/API/#new-redisport-host-options)) | | `UI` | | `bull-board` | UI to use (supported: `arena`, `bull-board`) | | `BULL_WATCH_QUEUE_PREFIXES` | | `bull` | Bull prefixes to monitor (globs like `prefix*` are supported) | | `BULL_COLLECT_QUEUE_METRICS_INTERVAL_MS` | | `60000` | How often queue metrics are gathered | diff --git a/bull_generator.ts b/bull_generator.ts index d8838e9..362c387 100644 --- a/bull_generator.ts +++ b/bull_generator.ts @@ -35,6 +35,7 @@ const config = cleanEnv(process.env, { REDIS_PASSWORD: str({ default: '' }), REDIS_PORT: port({ default: 6001 }), CREATE_DELAY_MS: num({ default: 0 }), + REDIS_FAMILY: num({ default: undefined }), CONCURRENCY: num({ default: 1 }), }); @@ -50,6 +51,7 @@ const main = async () => { host: config.REDIS_HOST, port: config.REDIS_PORT, password: config.REDIS_PASSWORD, + family: config.REDIS_FAMILY, }, }); @@ -59,6 +61,7 @@ const main = async () => { host: config.REDIS_HOST, port: config.REDIS_PORT, password: config.REDIS_PASSWORD, + family: config.REDIS_FAMILY, }, }); @@ -92,6 +95,7 @@ const main = async () => { host: config.REDIS_HOST, port: config.REDIS_PORT, password: config.REDIS_PASSWORD, + family: config.REDIS_FAMILY, }, limiter: { duration: config.LIMITER_DURATION_MS, diff --git a/environments/local.env b/environments/local.env index 74aea11..60fa570 100644 --- a/environments/local.env +++ b/environments/local.env @@ -1,4 +1,4 @@ -REDIS_PORT=6001 +REDIS_PORT=6379 REDIS_HOST=127.0.0.1 LOG_LABEL=bull-monitor LOG_LEVEL=debug @@ -8,4 +8,4 @@ COLLECT_NODEJS_METRICS=false BULL_WATCH_QUEUE_PREFIXES=bull BULL_COLLECT_QUEUE_METRICS_INTERVAL_MS=5000 UI=bull-board -VERSION=local \ No newline at end of file +VERSION=local diff --git a/package-lock.json b/package-lock.json index 79a3771..3734d11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "bull-arena": "^3.29.3", "bull-master": "^1.0.5", "bull-prom": "ejhayes/bull-prom#master", - "bullmq": "^1.86.2", + "bullmq": "^3.11.0", "commander": "^9.3.0", "envalid": "^7.3.1", "forever-monitor": "^3.0.3", @@ -4298,6 +4298,89 @@ "node": ">=10" } }, + "node_modules/bull-master/node_modules/bullmq": { + "version": "1.91.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.91.1.tgz", + "integrity": "sha512-u7dat9I8ZwouZ651AMZkBSvB6NVUPpnAjd4iokd9DM41whqIBnDjuL11h7+kEjcpiDKj6E+wxZiER00FqirZQg==", + "optional": true, + "dependencies": { + "cron-parser": "^4.6.0", + "get-port": "6.1.2", + "glob": "^8.0.3", + "ioredis": "^5.2.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "optional": true, + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "optional": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "optional": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "optional": true, + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "optional": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bull-master/node_modules/bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -4654,14 +4737,13 @@ } }, "node_modules/bullmq": { - "version": "1.90.2", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.90.2.tgz", - "integrity": "sha512-GBCvEdMG2mnR0Jkgc/Bv6ZBRbswkRopHJsyJik65/VXVO2/2jb8HC0+iC0cALmn7aCKpJGoVwtUmH0FLw7yLYQ==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.11.0.tgz", + "integrity": "sha512-l9/Wy1rKGQGgnlkKM50q6jknzEf2v9lhIh4xva71TqopWORUuGBtFDrHAQn/X6m+fQnXWiG5FVL0fnHdHhcQeA==", "dependencies": { "cron-parser": "^4.6.0", - "get-port": "6.1.2", "glob": "^8.0.3", - "ioredis": "^5.2.2", + "ioredis": "^5.3.0", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "semver": "^7.3.7", @@ -4669,17 +4751,6 @@ "uuid": "^9.0.0" } }, - "node_modules/bullmq/node_modules/get-port": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", - "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/bullmq/node_modules/uuid": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", @@ -8330,14 +8401,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", - "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -9878,9 +9949,9 @@ } }, "node_modules/luxon": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.0.3.tgz", - "integrity": "sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==", "engines": { "node": ">=12" } @@ -17696,6 +17767,69 @@ "uuid": "^8.3.0" } }, + "bullmq": { + "version": "1.91.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.91.1.tgz", + "integrity": "sha512-u7dat9I8ZwouZ651AMZkBSvB6NVUPpnAjd4iokd9DM41whqIBnDjuL11h7+kEjcpiDKj6E+wxZiER00FqirZQg==", + "optional": true, + "requires": { + "cron-parser": "^4.6.0", + "get-port": "6.1.2", + "glob": "^8.0.3", + "ioredis": "^5.2.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + }, + "dependencies": { + "cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "optional": true, + "requires": { + "luxon": "^3.2.1" + } + }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "optional": true + }, + "get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "optional": true + }, + "ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "optional": true, + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "optional": true + } + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -17967,14 +18101,13 @@ "requires": {} }, "bullmq": { - "version": "1.90.2", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.90.2.tgz", - "integrity": "sha512-GBCvEdMG2mnR0Jkgc/Bv6ZBRbswkRopHJsyJik65/VXVO2/2jb8HC0+iC0cALmn7aCKpJGoVwtUmH0FLw7yLYQ==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.11.0.tgz", + "integrity": "sha512-l9/Wy1rKGQGgnlkKM50q6jknzEf2v9lhIh4xva71TqopWORUuGBtFDrHAQn/X6m+fQnXWiG5FVL0fnHdHhcQeA==", "requires": { "cron-parser": "^4.6.0", - "get-port": "6.1.2", "glob": "^8.0.3", - "ioredis": "^5.2.2", + "ioredis": "^5.3.0", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "semver": "^7.3.7", @@ -17982,11 +18115,6 @@ "uuid": "^9.0.0" }, "dependencies": { - "get-port": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", - "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==" - }, "uuid": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", @@ -20825,14 +20953,14 @@ "requires": {} }, "ioredis": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", - "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -22003,9 +22131,9 @@ } }, "luxon": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.0.3.tgz", - "integrity": "sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w==" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==" }, "macos-release": { "version": "2.5.0", diff --git a/package.json b/package.json index d247192..f85439b 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "bull-arena": "^3.29.3", "bull-master": "^1.0.5", "bull-prom": "ejhayes/bull-prom#master", - "bullmq": "^1.86.2", + "bullmq": "^3.11.0", "commander": "^9.3.0", "envalid": "^7.3.1", "forever-monitor": "^3.0.3", diff --git a/src/bull/bull-queues.service.ts b/src/bull/bull-queues.service.ts index 5134339..e65580a 100644 --- a/src/bull/bull-queues.service.ts +++ b/src/bull/bull-queues.service.ts @@ -2,7 +2,7 @@ import { ConfigService } from '@app/config/config.service'; import { InjectLogger, LoggerService } from '@app/logger'; import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Mutex, withTimeout } from 'async-mutex'; -import { Queue, QueueScheduler } from 'bullmq'; +import { Queue } from 'bullmq'; import { RedisService } from 'nestjs-redis'; import { TypedEmitter } from 'tiny-typed-emitter2'; import { @@ -39,7 +39,6 @@ const REDIS_CONFIG_NOTIFY_KEYSPACE_EVENTS_FLAGS = 'A$K'; export class BullQueuesService implements OnModuleInit, OnModuleDestroy { private _initialized = false; private readonly _queues: { [queueName: string]: Queue } = {}; - private readonly _schedulers: { [queueName: string]: QueueScheduler } = {}; private readonly _redisMutex = withTimeout(new Mutex(), 10000); private readonly _bullMutex = withTimeout(new Mutex(), 10000); @@ -106,6 +105,7 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { host: this.configService.config.REDIS_HOST, port: this.configService.config.REDIS_PORT, password: this.configService.config.REDIS_PASSWORD, + family: this.configService.config.REDIS_FAMILY, }, }); this._queues[queueKey].on('error', (err) => { @@ -116,29 +116,6 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { this._queues[queueKey].on('ioredis:close', () => { this.removeQueue(queuePrefix, queueName); }); - /** - * From: https://docs.bullmq.io/guide/connections - * - * Every class will consume at least one Redis connection, but it - * is also possible to reuse connections in some situations. For example, - * the Queue and Worker classes can accept an existing ioredis instance, and - * by that reusing that connection, however QueueScheduler and QueueEvents - * cannot do that because they require blocking connections to Redis, which - * makes it impossible to reuse them. - */ - this._schedulers[queueKey] = new QueueScheduler(queueName, { - prefix: queuePrefix, - connection: { - host: this.configService.config.REDIS_HOST, - port: this.configService.config.REDIS_PORT, - password: this.configService.config.REDIS_PASSWORD, - }, - }); - this._schedulers[queueKey].on('error', (err) => { - Error.captureStackTrace(err); - this.logger.error(err.stack); - this.removeQueue(queuePrefix, queueName); - }); this.eventEmitter.emit( EVENT_TYPES.QUEUE_CREATED, new QueueCreatedEvent(queuePrefix, this._queues[queueKey]), @@ -156,14 +133,12 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { try { await this._queues[queueKey].close(); - await this._schedulers[queueKey].close(); } catch (err) { // in the event of an error just ignore it and move on this.logger.error(err); } delete this._queues[queueKey]; - delete this._schedulers[queueKey]; this.eventEmitter.emit( EVENT_TYPES.QUEUE_REMOVED, @@ -468,10 +443,7 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { this.eventEmitter.removeAllListeners(); // close all connections - for (const queue of [ - Object.values(this._queues), - Object.values(this._schedulers), - ].flat()) { + for (const queue of [Object.values(this._queues)].flat()) { this.logger.warn(`Closing queue: ${queue.name}`); await new Promise(async (resolve) => { (await queue.client).on('close', () => { diff --git a/src/bull/bull.module.ts b/src/bull/bull.module.ts index b0f60c3..b14995c 100644 --- a/src/bull/bull.module.ts +++ b/src/bull/bull.module.ts @@ -24,6 +24,7 @@ import { BullMQMetricsFactory } from './bullmq-metrics.factory'; password: configService.config.REDIS_PASSWORD, port: configService.config.REDIS_PORT, db: configService.config.REDIS_DB, + family: configService.config.REDIS_FAMILY, enableReadyCheck: true, reconnectOnError: () => true, }; diff --git a/src/bull/bullmq-metrics.factory.ts b/src/bull/bullmq-metrics.factory.ts index 44a74ac..92d537b 100644 --- a/src/bull/bullmq-metrics.factory.ts +++ b/src/bull/bullmq-metrics.factory.ts @@ -194,6 +194,7 @@ export class BullMQMetricsFactory { host: this.configService.config.REDIS_HOST, port: this.configService.config.REDIS_PORT, password: this.configService.config.REDIS_PASSWORD, + family: this.configService.config.REDIS_FAMILY, }, }); diff --git a/src/config/config.service.ts b/src/config/config.service.ts index 579073a..876079d 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -39,6 +39,10 @@ export class ConfigService { * Redis password (if needed) */ REDIS_PASSWORD: str({ default: '' }), + /** + * Redis family (if needed) + */ + REDIS_FAMILY: num({ default: undefined }), /** * Comma separate list of bull queue prefixes to * monitor (default: bull)