From 8465a7b7e9c947cb28a43c72abe6a9a6deaff92d Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 12:34:16 +0100 Subject: [PATCH] test backbeat config validation Issue: BB-623 --- lib/Config.js | 9 + tests/unit/lib/config/Config.spec.js | 25 ++ tests/unit/lib/config/config.json | 364 +++++++++++++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 tests/unit/lib/config/Config.spec.js create mode 100644 tests/unit/lib/config/config.json diff --git a/lib/Config.js b/lib/Config.js index dba169f4f..290560576 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -55,6 +55,15 @@ class Config extends EventEmitter { throw new Error(`could not parse config file: ${err.message}`); } + this._parseConfig(config); + } + + /** + * Parses Backbeat's configuration + * @param {Object} config backbeat configuration + * @returns {undefined} + */ + _parseConfig(config) { const parsedConfig = joi.attempt(config, backbeatConfigJoi); if (parsedConfig.extensions) { diff --git a/tests/unit/lib/config/Config.spec.js b/tests/unit/lib/config/Config.spec.js new file mode 100644 index 000000000..f3160d20a --- /dev/null +++ b/tests/unit/lib/config/Config.spec.js @@ -0,0 +1,25 @@ +'use strict'; // eslint-disable-line + +const assert = require('assert'); + +const { Config } = require('../../../../lib/Config'); +const backbeatConfig = require('./config.json'); + +describe('Config', () => { + it('should make the probeserver config in the queuePoulator' + + 'required when multiple extensions are configured', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + assert.throws(() => config._parseConfig(testConfig)); + }); + + it('should make the probeserver config in the queuePoulator' + + 'optional when only notification config is specified', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + testConfig.extensions = { notification: testConfig.extensions.notification }; + assert.doesNotThrow(() => config._parseConfig(testConfig)); + }); +}); diff --git a/tests/unit/lib/config/config.json b/tests/unit/lib/config/config.json new file mode 100644 index 000000000..57b697014 --- /dev/null +++ b/tests/unit/lib/config/config.json @@ -0,0 +1,364 @@ +{ + "zookeeper": { + "connectionString": "127.0.0.1:2181/backbeat", + "autoCreateNamespace": false + }, + "kafka": { + "hosts": "127.0.0.1:9092", + "backlogMetrics": { + "zkPath": "/backbeat/run/kafka-backlog-metrics", + "intervalS": 60 + }, + "maxRequestSize": 5000020 + }, + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "vaultAdmin": { + "host": "127.0.0.1", + "port": 8500 + }, + "replicationGroupId": "RG001 ", + "queuePopulator": { + "cronRule": "*/5 * * * * *", + "batchMaxRead": 10000, + "batchTimeoutMs": 9000, + "zookeeperPath": "/queue-populator", + "logSource": "mongo", + "bucketd": { + "host": "127.0.0.1", + "port": 9000 + }, + "dmd": { + "host": "127.0.0.1", + "port": 9990 + }, + "mongo": { + "replicaSetHosts": + "localhost:27017,localhost:27018,localhost:27019", + "writeConcern": "majority", + "replicaSet": "rs0", + "readPreference": "primary", + "database": "metadata" + }, + "kafka": { + "topic": "backbeat-oplog", + "consumerGroupId": "backbeat-qp-oplog-group" + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 4042 + } + }, + "extensions": { + "ingestion": { + "auth": { + "type": "service", + "account": "service-md-ingestion" + }, + "topic": "backbeat-ingestion", + "zookeeperPath": "/ingestion", + "cronRule": "*/5 * * * * *", + "maxParallelReaders": 5, + "sources": [ + { + "name": "zenko-bucket", + "bucket": "src-bucket", + "host": "localhost", + "port": 8000, + "https": false, + "type": "scality_s3", + "locationConstraint": "a-zenko-location", + "auth": { + "accessKey": "myAccessKey", + "secretKey": "myEncryptedSecretKey" + } + } + ], + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8550 + } + }, + "mongoProcessor": { + "topic": "backbeat-ingestion", + "groupId": "backbeat-mongo-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8551 + } + }, + "replication": { + "source": { + "transport": "http", + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "auth": { + "type": "service", + "account": "service-replication", + "vault": { + "host": "127.0.0.1", + "port": 8500, + "adminPort": 8600 + } + } + }, + "destination": { + "transport": "http", + "bootstrapList": [ + { "site": "zenko", "servers": ["localhost:8001"], + "echo": false }, + { "site": "wontwork-location", "type": "aws_s3" }, + { "site": "aws-location", "type": "aws_s3" } + ], + "auth": { + "type": "service", + "account": "service-replication" + } + }, + "topic": "backbeat-replication", + "dataMoverTopic": "backbeat-data-mover", + "replicationStatusTopic": "backbeat-replication-status", + "replicationFailedTopic": "backbeat-replication-failed", + "monitorReplicationFailures": true, + "monitorReplicationFailureExpiryTimeS": 86400, + "replayTopics": [ + { + "topicName": "backbeat-replication-replay-0", + "retries": 5 + } + ], + "queueProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "aws_s3": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "azure": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "gcp": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "scality": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + } + }, + "concurrency": 10, + "mpuPartsConcurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4043 + } + }, + "replicationStatusProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4045 + } + }, + "objectSizeMetrics": [ + 66560, + 8388608, + 68157440 + ] + }, + "lifecycle": { + "auth": { + "type": "service", + "account": "service-lifecycle" + }, + "zookeeperPath": "/lifecycle", + "bucketTasksTopic": "backbeat-lifecycle-bucket-tasks", + "objectTasksTopic": "backbeat-lifecycle-object-tasks", + "transitionTasksTopic": "backbeat-lifecycle-transition-tasks", + "conductor": { + "cronRule": "0 */5 * * * *", + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8552 + } + }, + "bucketProcessor": { + "groupId": "backbeat-lifecycle-bucket-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8553 + } + }, + "objectProcessor": { + "groupId": "backbeat-lifecycle-object-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8554 + } + }, + "coldStorageArchiveTopicPrefix": "cold-archive-req-", + "coldStorageRestoreTopicPrefix": "cold-restore-req-", + "coldStorageGCTopicPrefix": "cold-gc-req-", + "coldStorageStatusTopicPrefix": "cold-status-" + }, + "gc": { + "topic": "backbeat-gc", + "auth": { + "type": "service", + "account": "service-gc" + }, + "consumer": { + "groupId": "backbeat-gc-consumer-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10 + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8555 + } + }, + "oplogPopulator": { + "topic": "backbeat-oplog", + "kafkaConnectHost": "127.0.0.1", + "kafkaConnectPort": 8083, + "numberOfConnectors": 1, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8556 + } + }, + "notification": { + "topic": "backbeat-bucket-notification", + "monitorNotificationFailures": true, + "queueProcessor": { + "groupId": "backbeat-bucket-notification-group", + "concurrency": 10 + }, + "destinations": [ + { + "resource": "destination1", + "type": "kafka", + "host": "localhost", + "port": 9092, + "topic": "destination-topic-1", + "internalTopic": "internal-notification-topic-destination1", + "auth": {} + }, + { + "resource": "destination2", + "type": "kafka", + "host": "localhost:9092", + "port": 9092, + "topic": "destination-topic-2", + "internalTopic": "internal-notification-topic-destination2", + "auth": {} + } + ] + } + }, + "log": { + "logLevel": "info", + "dumpLevel": "error" + }, + "metrics": { + "topic": "backbeat-metrics" + }, + "server": { + "healthChecks": { + "allowFrom": ["127.0.0.1/8", "::1"] + }, + "host": "127.0.0.1", + "port": 8900 + }, + "redis": { + "host": "localhost", + "port": 6379 + }, + "certFilePaths": { + "key": "", + "cert": "", + "ca": "" + } +}