Improvement/bb 632 bump mongo db driver 6

.github/actions/debug-wait/action.yaml

@@ -0,0 +1,15 @@
+---
+name: "Debug wait"
+description: "If debugging is enabled, wait"
+
+runs:
+  using: composite
+  steps:
+    - name: "Debug: SSH to runner"
+      uses: scality/actions/[email protected]
+      continue-on-error: true
+      with:
+        tmate-server-host: ${{ env.TMATE_SERVER_HOST }}
+        tmate-server-port: ${{ env.TMATE_SERVER_PORT }}
+        tmate-server-rsa-fingerprint: ${{ env.TMATE_SERVER_RSA_FINGERPRINT }}
+        tmate-server-ed25519-fingerprint: ${{ env.TMATE_SERVER_ED25519_FINGERPRINT }}

.github/workflows/tests.yaml

@@ -3,6 +3,11 @@ name: tests
 
 env:
   GINKGO_VERSION: v1.15.2
+  # DEBUG WAIT
+  TMATE_SERVER_HOST: ${{ secrets.TMATE_SERVER_HOST }}
+  TMATE_SERVER_PORT: ${{ secrets.TMATE_SERVER_PORT }}
+  TMATE_SERVER_RSA_FINGERPRINT: ${{ secrets.TMATE_SERVER_RSA_FINGERPRINT }}
+  TMATE_SERVER_ED25519_FINGERPRINT: ${{ secrets.TMATE_SERVER_ED25519_FINGERPRINT }}
 
 on:
   push:
@@ -192,3 +197,9 @@ jobs:
         # will crash if they end up in memory all at the same time (circuit breaking
         # ineffective) while waiting to be committed to the kafka topic.
         NODE_OPTIONS: '--max-old-space-size=150'
+
+    - name: Debug wait
+      uses: ./.github/actions/debug-wait
+      timeout-minutes: 60
+      if: failure() && runner.debug == '1'
+

extensions/notification/configManager/MongoConfigManager.js

@@ -97,18 +97,13 @@ class MongoConfigManager extends BaseConfigManager {
      */
     _setupMongoClient(cb) {
         const mongoUrl = constructConnectionString(this._mongoConfig);
-        MongoClient.connect(mongoUrl, {
+        const client = new MongoClient(mongoUrl, {
             replicaSet: this._mongoConfig.replicaSet,
             useNewUrlParser: true,
-        },
-        (err, client) => {
-            if (err) {
-                this._logger.error('Could not connect to MongoDB', {
-                    method: 'MongoConfigManager._setupMongoClient',
-                    error: err.message,
-                });
-                return cb(err);
-            }
+            readPreference: this._mongoConfig.readPreference,
+        });
+
+        return client.connect().then(client => {
             this._logger.debug('Connected to MongoDB', {
                 method: 'MongoConfigManager._setupMongoClient',
             });
@@ -133,6 +128,12 @@ class MongoConfigManager extends BaseConfigManager {
             } catch (error) {
                 return cb(error);
             }
+        }).catch(err => {
+            this._logger.error('Could not connect to MongoDB', {
+                method: 'MongoConfigManager._setupMongoClient',
+                error: err.message,
+            });
+            return cb(err);
         });
     }
 

extensions/oplogPopulator/OplogPopulator.js

@@ -83,11 +83,13 @@ class OplogPopulator {
      */
     async _setupMongoClient() {
         try {
-            const client = await MongoClient.connect(this._mongoUrl, {
+            let client = new MongoClient(this._mongoUrl, {
                 replicaSet: this._replicaSet,
                 useNewUrlParser: true,
                 useUnifiedTopology: true,
+                readPreference: this._mongoConfig.readPreference,
             });
+            client = await client.connect();
             // connect to metadata DB
             this._mongoClient = client.db(this._database, {
                 ignoreUndefined: true,

extensions/replication/tasks/ReplicateObject.js

@@ -343,7 +343,7 @@ class ReplicateObject extends BackbeatTask {
         };
         return this.backbeatSource.getMetadata(params, (err, blob) => {
             if (err) {
-                err.origin = 'source';
+                err.origin = 'source'; // eslint-disable-line no-param-reassign
                 log.error('error getting metadata blob from S3', {
                     method: 'ReplicateObject._refreshSourceEntry',
                     error: err,

extensions/utils/LocationStatusStream.js

@@ -63,18 +63,12 @@
      */
     _setupMongoClient(cb) {
         const mongoUrl = constructConnectionString(this._mongoConfig);
-        MongoClient.connect(mongoUrl, {
+        const client = new MongoClient(mongoUrl, {
                 replicaSet: this._mongoConfig.replicaSet,
                 useNewUrlParser: true,
                 useUnifiedTopology: true,
-        }, (err, client) => {
-            if (err) {
-                this._log.error('Could not connect to MongoDB', {
-                    method: 'ServiceStatusManager._setupMongoClient',
-                    error: err.message,
-                });
-                return cb(err);
-            }
+        });
+        return client.connect().then(client => {
             // connect to metadata DB
             this._mongoClient = client.db(this._mongoConfig.database, {
                 ignoreUndefined: true,
@@ -96,6 +90,12 @@
                 return cb();
             });
             return undefined;
+        }).catch(err => {
+            this._log.error('Could not connect to MongoDB', {
+                method: 'ServiceStatusManager._setupMongoClient',
+                error: err.message,
+            });
+            return cb(err);
         });
     }
 
@@ -106,22 +106,23 @@
      */
     _initializeLocationStatuses(cb) {
         this._locationStatusColl.find({})
-            .toArray((err, locations) => {
-                if (err) {
-                    this._log.error('Could not fetch location statuses from mongo', {
-                        method: 'ServiceStatusManager._initializeLocationStatuses',
-                        error: err.message,
-                    });
-                    return cb(err);
+        .toArray()
+        .then(locations => {
+            locations.forEach(location => {
+                const isPaused = location.value[this._serviceName].paused;
+                if (isPaused) {
+                    this._pauseServiceForLocation(location._id);
                 }
-                locations.forEach(location => {
-                    const isPaused = location.value[this._serviceName].paused;
-                    if (isPaused) {
-                        this._pauseServiceForLocation(location._id);
-                    }
-                });
-                return cb();
             });
+            return cb();
+        })
+        .catch(err => {
+            this._log.error('Could not fetch location statuses from mongo', {
+                method: 'ServiceStatusManager._initializeLocationStatuses',
+                error: err.message,
+            });
+            return cb(err);
+        });
     }
 
     /**

lib/api/BackbeatAPI.js

@@ -1335,21 +1335,14 @@ class BackbeatAPI {
             });
             return cb();
         } else {
-            // To be removed once the mongodb drivers are bumped
-            // BB-585
             const mongoUrl = constructConnectionString(mongoConfig);
-            return MongoClient.connect(mongoUrl, {
+            const client = new MongoClient(mongoUrl, {
                 replicaSet: mongoConfig.replicaSet,
                 useNewUrlParser: true,
                 useUnifiedTopology: true,
-            }, (err, client) => {
-                if (err) {
-                    this._logger.error('Could not connect to MongoDB', {
-                        method: 'BackbeatAPI._setupMongoClient',
-                        error: err.message,
-                    });
-                    return cb(err);
-                }
+            });
+
+            return client.connect().then(client => {
                 // connect to metadata DB
                 this._mongoClient = client.db(mongoConfig.database, {
                     ignoreUndefined: true,
@@ -1358,6 +1351,12 @@ class BackbeatAPI {
                     method: 'BackbeatAPI._setupMongoClient',
                 });
                 return cb();
+            }).catch(err => {
+                this._logger.error('Could not connect to MongoDB', {
+                    method: 'BackbeatAPI._setupMongoClient',
+                    error: err.message,
+                });
+                return cb(err);
             });
         }
     }

lib/util/LocationStatusManager.js

@@ -68,19 +68,23 @@
      * @return {undefined}
      */
     _initCollection(cb) {
-        this._mongoClient.createCollection(locationStatusCollection, err => {
-            // in case the collection already exists, we ignore the error
-            if (err && err.codeName !== 'NamespaceExists') {
-                this._logger.error('Could not create mongo collection', {
-                    method: 'LocationStatusManager._initCollection',
-                    collection: locationStatusCollection,
-                    error: err.message,
-                });
-                return cb(err);
-            }
-            this._locationStatusColl = this._mongoClient.collection(locationStatusCollection);
-            return cb();
-        });
+        return this._mongoClient.createCollection(locationStatusCollection)
+            .then(() => {
+                this._locationStatusColl = this._mongoClient.collection(locationStatusCollection);
+                return cb();
+            })
+            .catch(err => {
+                if (err.codeName !== 'NamespaceExists') {
+                    this._logger.error('Could not create mongo collection', {
+                        method: 'LocationStatusManager._initCollection',
+                        collection: locationStatusCollection,
+                        error: err.message,
+                    });
+                    return cb(err);
+                }
+                this._locationStatusColl = this._mongoClient.collection(locationStatusCollection);
+                return cb();
+            });
     }
 
     /**
@@ -89,16 +93,17 @@
      * @returns {undefined}}
      */
     _listCollectionDocuments(cb) {
-        this._locationStatusColl.find({}, (err, cursor) => {
-            if (err) {
+        this._locationStatusColl.find({})
+            .toArray()
+            .then(docs => cb(null, docs))
+            .catch(err => {
                 this._logger.error('Could not list documents', {
                     method: 'LocationStatusManager._listCollectionDocuments',
                     collection: locationStatusCollection,
                     error: err.message,
                 });
-            }
-            return cursor.toArray(cb);
-        });
+                cb(err);
+            });
     }
 
     /**
@@ -139,15 +144,14 @@
             _id: {
                 $in: invalidLocations
             }
-        }, err => {
-            if (err) {
-                this._logger.error('Could not delete invalid locations', {
-                    method: 'LocationStatusManager._deleteInvalidLocations',
-                    error: err.message,
-                });
-                return cb(err);
-            }
-            return cb(null, locations);
+        })
+        .then(() => cb(null, locations))
+        .catch(err => {
+            this._logger.error('Could not delete invalid locations', {
+                method: 'LocationStatusManager._deleteInvalidLocations',
+                error: err.message,
+            });
+            cb(err);
         });
     }
 
@@ -166,10 +170,10 @@
         async.eachLimit(newLocations, 10, (location, next) => {
             const locationConfig = new LocationStatus(this._supportedServices);
             this._locationStatusStore[location] = locationConfig;
-            this._locationStatusColl.insert({
+            this._locationStatusColl.insertOne({
                 _id: location,
                 value: locationConfig.getValue(),
-            }, next);
+            }).finally(next);
         }, err => {
             if (err) {
                 this._logger.error('Could not add new locations', {

package.json

@@ -17,7 +17,7 @@
     "oplog_populator": "node extensions/oplogPopulator/OplogPopulatorTask.js",
     "mongo_queue_processor": "node extensions/mongoProcessor/mongoProcessorTask.js",
     "garbage_collector": "node extensions/gc/service.js",
-    "test": "mocha --recursive tests/unit/oplogPopulator --timeout 30000",
+    "test": "mocha --recursive tests/unit --timeout 30000",
     "cover:test": "nyc --clean --silent yarn run test && nyc report --report-dir ./coverage/test --reporter=lcov",
     "ft_test": "mocha --recursive $(find tests/functional -name '*.js') --timeout 30000",
     "ft_test:notification": "mocha --recursive $(find tests/functional/notification -name '*.js') --timeout 30000",
@@ -59,7 +59,7 @@
     "ioredis": "^4.9.5",
     "joi": "^17.6.0",
     "minimatch": "^3.0.4",
-    "mongodb": "^3.1.13",
+    "mongodb": "^6.11.0",
     "node-forge": "^0.7.6",
     "node-rdkafka": "^2.12.0",
     "node-rdkafka-prometheus": "^1.0.0",

tests/config.json

@@ -23,7 +23,8 @@
         },
         "mongo": {
             "logName": "s3-recordlog",
-            "replicaSetHosts": "localhost:27018"
+            "replicaSetHosts": "localhost:27018",
+            "readPreference": "primary"
         },
         "probeServer": {
             "bindAddress": "localhost",