Merge branch 'improvement/BB-436' into q/8.6

.github/workflows/alerts.yaml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Render and test lifecycle 
         uses: scality/[email protected]

.github/workflows/docker-build.yaml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildk
         uses: docker/setup-buildx-action@v2

.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # TODO: remove the following step once Oras CLI 0.13.0 bug https://github.com/oras-project/oras/issues/447 is fixed.
       - name: Downgrade Oras to 0.12.0

.github/workflows/tests.yaml

@@ -19,7 +19,7 @@ jobs:
       packages: write
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Docker Buildk
       uses: docker/setup-buildx-action@v2
@@ -53,7 +53,7 @@ jobs:
         cache-to: type=gha,mode=max
 
     - name: Build and push MongoDB
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v4
       with:
         push: true
         context: .github/dockerfiles/mongodb
@@ -92,7 +92,7 @@ jobs:
         - 27019:27019
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install build dependencies
       run: |
         sudo apt-get update

conf/config.json

@@ -8,7 +8,8 @@
         "backlogMetrics": {
             "zkPath": "/backbeat/run/kafka-backlog-metrics",
             "intervalS": 60
-        }
+        },
+        "maxRequestSize": 5000020
     },
     "s3": {
         "host": "127.0.0.1",

extensions/gc/GarbageCollectorProducer.js

@@ -25,6 +25,7 @@ class GarbageCollectorProducer {
     setupProducer(cb) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic: this._topic,
         });
         producer.once('error', () => {});

extensions/lifecycle/LifecycleQueuePopulator.js

@@ -54,6 +54,7 @@ class LifecycleQueuePopulator extends QueuePopulatorExtension {
         }
         const producer = new BackbeatProducer({
             kafka: { hosts: this.kafkaConfig.hosts },
+            maxRequestSize: this.kafkaConfig.maxRequestSize,
             topic,
         });
         producer.once('error', done);

extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js

@@ -363,6 +363,7 @@ class LifecycleBucketProcessor {
     _setupProducer(cb) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic: this._lcConfig.objectTasksTopic,
         });
         producer.once('error', err => {

extensions/lifecycle/conductor/LifecycleConductor.js

@@ -779,6 +779,7 @@ class LifecycleConductor {
     _setupProducer(cb) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this.kafkaConfig.hosts },
+            maxRequestSize: this.kafkaConfig.maxRequestSize,
             topic: this.lcConfig.bucketTasksTopic,
         });
         producer.once('error', cb);

extensions/oplogPopulator/OplogPopulator.js

@@ -9,10 +9,12 @@ const Allocator = require('./modules/Allocator');
 const ConnectorsManager = require('./modules/ConnectorsManager');
 const { ZenkoMetrics } = require('arsenal').metrics;
 const OplogPopulatorMetrics = require('./OplogPopulatorMetrics');
+const { OplogPopulatorConfigJoiSchema } = require('./OplogPopulatorConfigValidator');
+const { mongoJoi } = require('../../lib/config/configItems.joi');
 
 const paramsJoi = joi.object({
-    config: joi.object().required(),
-    mongoConfig: joi.object().required(),
+    config: OplogPopulatorConfigJoiSchema.required(),
+    mongoConfig: mongoJoi.required(),
     activeExtensions: joi.array().required(),
     logger: joi.object().required(),
     enableMetrics: joi.boolean().default(true),

extensions/oplogPopulator/OplogPopulatorConfigValidator.js

@@ -17,4 +17,7 @@ function configValidator(backbeatConfig, extConfig) {
     return validatedConfig;
 }
 
-module.exports = configValidator;
+module.exports = {
+    OplogPopulatorConfigJoiSchema: joiSchema,
+    OplogPopulatorConfigValidator: configValidator
+};

extensions/oplogPopulator/index.js

@@ -1,4 +1,4 @@
-const OplogPopulatorConfigValidator =
+const { OplogPopulatorConfigValidator } =
     require('./OplogPopulatorConfigValidator');
 
 module.exports = {

extensions/replication/failedCRR/FailedCRRProducer.js

@@ -27,6 +27,7 @@ class FailedCRRProducer {
     setupProducer(cb) {
         this._producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic: this._topic,
         });
         this._producer.once('error', () => {});

extensions/replication/queueProcessor/QueueProcessor.js

@@ -371,6 +371,7 @@ class QueueProcessor extends EventEmitter {
     _setupProducer(done) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this.kafkaConfig.hosts },
+            maxRequestSize: this.kafkaConfig.maxRequestSize,
             topic: this.repConfig.replicationStatusTopic,
         });
         producer.once('error', done);

extensions/replication/replay/ReplayProducer.js

@@ -27,6 +27,7 @@ class ReplayProducer {
     setupProducer(cb) {
         this._producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic: this._topic,
         });
         this._producer.once('error', () => {});

extensions/replication/replicationStatusProcessor/ReplicationStatusProcessor.js

@@ -386,6 +386,7 @@ class ReplicationStatusProcessor {
                     this.bucketNotificationConfig.topic;
                 const producer = new BackbeatProducer({
                     kafka: { hosts: this.kafkaConfig.hosts },
+                    maxRequestSize: this.kafkaConfig.maxRequestSize,
                     topic: internalTopic,
                 });
                 producer.once('error', done);

lib/BackbeatProducer.js

@@ -10,7 +10,7 @@ const KafkaBacklogMetrics = require('./KafkaBacklogMetrics');
 const Constants = require('./constants');
 
 const DEFAULT_POLL_INTERVAL = 2000;
-const PRODUCER_MESSAGE_MAX_BYTES = 5000020;
+const { KAFKA_PRODUCER_MESSAGE_MAX_BYTES } = require('./config.joi');
 const FLUSH_TIMEOUT = 5000;
 
 
@@ -45,7 +45,7 @@ class BackbeatProducer extends EventEmitter {
                 }).required(),
                 topic: joi.string(),
                 pollIntervalMs: joi.number().default(DEFAULT_POLL_INTERVAL),
-                messageMaxBytes: joi.number().default(PRODUCER_MESSAGE_MAX_BYTES),
+                maxRequestSize: joi.number().default(KAFKA_PRODUCER_MESSAGE_MAX_BYTES),
             }
         );
     }
@@ -65,7 +65,7 @@ class BackbeatProducer extends EventEmitter {
     get producerConfig() {
         return {
             'metadata.broker.list': this._kafkaHosts,
-            'message.max.bytes': this._messageMaxBytes,
+            'message.max.bytes': this._maxRequestSize,
             'dr_cb': true,
             'compression.type': Constants.compressionType,
         };
@@ -103,12 +103,12 @@ class BackbeatProducer extends EventEmitter {
             kafka,
             topic,
             pollIntervalMs,
-            messageMaxBytes,
+            maxRequestSize,
         } = joiResult;
         this._kafkaHosts = kafka.hosts;
         this._topic = topic && withTopicPrefix(topic);
         this._pollIntervalMs = pollIntervalMs;
-        this._messageMaxBytes = messageMaxBytes;
+        this._maxRequestSize = maxRequestSize;
     }
 
     connect() {

lib/Config.js

@@ -8,7 +8,7 @@ const path = require('path');
 const crypto = require('crypto');
 
 const extensions = require('../extensions');
-const backbeatConfigJoi = require('./config.joi');
+const { backbeatConfigJoi } = require('./config.joi');
 
 const locationTypeMatch = {
     'location-mem-v1': 'mem',

lib/MetricsProducer.js

@@ -23,6 +23,7 @@ class MetricsProducer {
     setupProducer(done) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic: this._topic,
         });
         producer.once('error', done);

lib/api/BackbeatAPI.js

@@ -1300,6 +1300,7 @@ class BackbeatAPI {
     _setProducer(topic, cb) {
         const producer = new BackbeatProducer({
             kafka: { hosts: this._kafkaConfig.hosts },
+            maxRequestSize: this._kafkaConfig.maxRequestSize,
             topic,
         });