From c62ef8cbc2df36060c110d2d54a0c388ba926051 Mon Sep 17 00:00:00 2001 From: Ido Date: Tue, 22 Mar 2022 16:28:12 +0200 Subject: [PATCH 1/2] support documentdb --- packages/external-db-config/lib/factory.js | 1 + .../lib/utils/config_utils.js | 2 +- .../lib/connection_provider.js | 4 +++- .../external-db-mongo/lib/documentdb_utils.js | 20 +++++++++++++++++++ .../velo-external-db/lib/storage/factory.js | 4 ++++ 5 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 packages/external-db-mongo/lib/documentdb_utils.js diff --git a/packages/external-db-config/lib/factory.js b/packages/external-db-config/lib/factory.js index 9f23fbdd3..ac36f72d5 100644 --- a/packages/external-db-config/lib/factory.js +++ b/packages/external-db-config/lib/factory.js @@ -23,6 +23,7 @@ const create = () => { internalConfigReader = new aws.AwsDynamoConfigReader(region, secretId) break case 'mongo': + case 'documentdb': internalConfigReader = new aws.AwsMongoConfigReader(region, secretId) break default: diff --git a/packages/external-db-config/lib/utils/config_utils.js b/packages/external-db-config/lib/utils/config_utils.js index 5a7df0361..a761a5086 100644 --- a/packages/external-db-config/lib/utils/config_utils.js +++ b/packages/external-db-config/lib/utils/config_utils.js @@ -2,7 +2,7 @@ const objectContainsKey = (obj, key) => typeof obj[key] === 'string' && obj[key] const checkRequiredKeys = (obj, requiredKeys) => requiredKeys.filter(key => !objectContainsKey(obj, key)) -const supportedDBs = ['postgres', 'spanner', 'firestore', 'mssql', 'mysql', 'mongo', 'airtable', 'dynamodb', 'bigquery', 'google-sheets'] +const supportedDBs = ['postgres', 'spanner', 'firestore', 'mssql', 'mysql', 'mongo', 'airtable', 'dynamodb', 'bigquery', 'google-sheets', 'documentdb'] const supportedVendors = ['gcp', 'aws', 'azure'] diff --git a/packages/external-db-mongo/lib/connection_provider.js b/packages/external-db-mongo/lib/connection_provider.js index 6245c1fd3..2ded61d51 100644 --- a/packages/external-db-mongo/lib/connection_provider.js +++ b/packages/external-db-mongo/lib/connection_provider.js @@ -4,9 +4,11 @@ const DataProvider = require('./mongo_data_provider') const FilterParser = require('./sql_filter_transformer') const DatabaseOperations = require('./mongo_operations') const { notConnectedPool, emptyClient } = require('./mongo_utils') +const { documentDBConnectionOptions } = require('./documentdb_utils') const init = async(cfg) => { - const client = cfg.connectionUri ? new MongoClient(cfg.connectionUri) : emptyClient() + const { connectionUri, options } = cfg.documentDb ? await documentDBConnectionOptions(cfg, __dirname) : { connectionUri: cfg.connectionUri, options: {} } + const client = connectionUri ? new MongoClient(connectionUri, options) : emptyClient() const { pool, cleanup } = await client.connect() .then((res) => { diff --git a/packages/external-db-mongo/lib/documentdb_utils.js b/packages/external-db-mongo/lib/documentdb_utils.js new file mode 100644 index 000000000..d729ac9ff --- /dev/null +++ b/packages/external-db-mongo/lib/documentdb_utils.js @@ -0,0 +1,20 @@ +const { URL } = require('url') +const https = require('https') +const fs = require('fs') + +const documentDBConnectionOptions = async(cfg, path) => { + const documentDbUri = new URL(cfg.connectionUri) + + if (documentDbUri.searchParams.get('ssl_ca_certs') === 'rds-combined-ca-bundle.pem') { + documentDbUri.searchParams.delete('ssl_ca_certs') + await https.get('https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem', resp => resp.pipe(fs.createWriteStream(`${path}/rds-combined-ca-bundle.pem`))) + return { + connectionUri: documentDbUri.toString(), + options: { tlsCAFile: `${path}/rds-combined-ca-bundle.pem` } + } + } + + return { connectionUri: documentDbUri.toString(), options: {} } +} + +module.exports = { documentDBConnectionOptions } \ No newline at end of file diff --git a/packages/velo-external-db/lib/storage/factory.js b/packages/velo-external-db/lib/storage/factory.js index 172bf737c..a9bdd01e2 100644 --- a/packages/velo-external-db/lib/storage/factory.js +++ b/packages/velo-external-db/lib/storage/factory.js @@ -53,6 +53,10 @@ const init = async(_type, vendor, config) => { const { init } = require('external-db-bigquery') return append(await init(cfg), cfg.secretKey) } + case 'documentdb': { + const { init } = require('external-db-mongo') + return append(await init({ ...cfg, documentDb: true }), cfg.secretKey) + } default: { const init = require('./stub-db/init') return append(await init(type), cfg.secretKey) From 54f99de8f073197d2d1ab06cd909c7651f1e18d6 Mon Sep 17 00:00:00 2001 From: Ido Date: Tue, 22 Mar 2022 17:26:45 +0200 Subject: [PATCH 2/2] resolve only after downloading the file --- .../external-db-mongo/lib/documentdb_utils.js | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/external-db-mongo/lib/documentdb_utils.js b/packages/external-db-mongo/lib/documentdb_utils.js index d729ac9ff..1fd3007c8 100644 --- a/packages/external-db-mongo/lib/documentdb_utils.js +++ b/packages/external-db-mongo/lib/documentdb_utils.js @@ -3,18 +3,23 @@ const https = require('https') const fs = require('fs') const documentDBConnectionOptions = async(cfg, path) => { - const documentDbUri = new URL(cfg.connectionUri) + return new Promise(resolve => { + const documentDbUri = new URL(cfg.connectionUri) - if (documentDbUri.searchParams.get('ssl_ca_certs') === 'rds-combined-ca-bundle.pem') { - documentDbUri.searchParams.delete('ssl_ca_certs') - await https.get('https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem', resp => resp.pipe(fs.createWriteStream(`${path}/rds-combined-ca-bundle.pem`))) - return { - connectionUri: documentDbUri.toString(), - options: { tlsCAFile: `${path}/rds-combined-ca-bundle.pem` } + if (documentDbUri.searchParams.get('ssl_ca_certs') === 'rds-combined-ca-bundle.pem') { + documentDbUri.searchParams.delete('ssl_ca_certs') + const file = fs.createWriteStream(`${path}/rds-combined-ca-bundle.pem`) + https.get('https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem', resp => resp.pipe(file)) + file.on('finish', () => resolve({ + connectionUri: documentDbUri.toString(), options: { tlsCAFile: `${path}/rds-combined-ca-bundle.pem` } + })) } - } - - return { connectionUri: documentDbUri.toString(), options: {} } + + else { + resolve( { connectionUri: documentDbUri.toString(), options: {} }) + } + }) + } module.exports = { documentDBConnectionOptions } \ No newline at end of file