From 9d00d6c272e48eaab28f2ed5a022b8ba65193f25 Mon Sep 17 00:00:00 2001 From: Prajwal Pai <108796209+prajwal-pai77@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:39:50 +0530 Subject: [PATCH] DA-355 Add Support for Mongodb to CB data migration (#356) Adds the UI for MongoDB to CB data migration. --- package-lock.json | 204 ++++++-- package.json | 19 +- src/commands/extensionCommands/commands.ts | 1 + src/commands/tools/dataImport.ts | 6 +- src/extension.ts | 26 +- src/handlers/handleCLIDownloader.ts | 225 ++++++-- src/pages/Tools/MdbMigrate/mdbMigrate.ts | 258 +++++++++ src/tools/MDBMigrate.ts | 74 +++ src/util/DependencyDownloaderUtils/CBTool.ts | 3 +- src/util/constants.ts | 1 + src/webViews/tools/mdbMigrate.webview.ts | 519 +++++++++++++++++++ 11 files changed, 1232 insertions(+), 104 deletions(-) create mode 100644 src/pages/Tools/MdbMigrate/mdbMigrate.ts create mode 100644 src/tools/MDBMigrate.ts create mode 100644 src/webViews/tools/mdbMigrate.webview.ts diff --git a/package-lock.json b/package-lock.json index cb31ac89..bb73bac3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "vscode-couchbase", - "version": "1.2.1", + "version": "1.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vscode-couchbase", - "version": "1.2.1", + "version": "1.2.2", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@chatscope/chat-ui-kit-styles": "^1.4.0", @@ -35,6 +35,7 @@ "gitly": "^2.4.2", "keytar": "^7.7.0", "monaco-editor": "^0.43.0", + "mongodb": "^6.4.0", "node-addon-api": "^6.0.0", "prop-types": "^15.7.2", "react": "^18.2.0", @@ -57,6 +58,7 @@ "@babel/traverse": "^7.22.8", "@types/glob": "^7.2.0", "@types/mocha": "^8.2.3", + "@types/mongodb": "^4.0.7", "@types/node": "^12.20.42", "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", @@ -2275,9 +2277,9 @@ "integrity": "sha512-016mBJD3DESw7Nh+lkKcPd22xG92ghA0VpIXIbjQtmXhC7Ve6wRazTy8z1Ahut+Tbv179+JxrftuMngsj/yV8Q==" }, "node_modules/@couchbase/couchbase-darwin-arm64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-arm64-napi/-/couchbase-darwin-arm64-napi-4.2.9.tgz", - "integrity": "sha512-XDRWSFKXGk2wsQbva7Zc+Atu1Xdqnj1EA9Xble7TsSDdN/6l0RO2t5uvRQYhZ5BBq0h8SFOD27e5Df1gHNTCEg==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-arm64-napi/-/couchbase-darwin-arm64-napi-4.3.0.tgz", + "integrity": "sha512-uRgthYNX544C5sODzhxs/8QmII+eymmvw16tncFOAss1Qp/o2PrTrzoZLCaIF+dHuq6L3DdLO7v/KvhO1+jUpQ==", "cpu": [ "arm64" ], @@ -2290,9 +2292,9 @@ } }, "node_modules/@couchbase/couchbase-darwin-x64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-x64-napi/-/couchbase-darwin-x64-napi-4.2.9.tgz", - "integrity": "sha512-URBW2V2XEDXXEC/Kaak3+EX+hnomQQ9YqPbOX65k1roy5HXuYXzdSgWv+b2bCRPIKP1ZvZDU14juvNTKRTQxnw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-x64-napi/-/couchbase-darwin-x64-napi-4.3.0.tgz", + "integrity": "sha512-HEqyyWxvWenW8nZ8qq4Q8f7b7ajLFJsfOTfFzyPWZsoiIJb4SPU9fiF3Ik7mmwweady4Vx5Gb4VwetjRLl6+7Q==", "cpu": [ "x64" ], @@ -2305,9 +2307,9 @@ } }, "node_modules/@couchbase/couchbase-linux-arm64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linux-arm64-napi/-/couchbase-linux-arm64-napi-4.2.9.tgz", - "integrity": "sha512-/krjBtwfSE1lmpG6/PluFFi7++z75GCOjSQ4Pdmn+txcwjpy7G9hu9NKxBRUn5gBYO7q/5tp0Z3LxdN2c7haWA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linux-arm64-napi/-/couchbase-linux-arm64-napi-4.3.0.tgz", + "integrity": "sha512-BVJDSb1IW49dWhDNGnz7+ljxb2NJw6sIGWqMkKmcJWRf40FY9uNScRw7D9Bdjxgoc941qXDwm0EuqYDjEX/wUQ==", "cpu": [ "arm64" ], @@ -2320,9 +2322,9 @@ } }, "node_modules/@couchbase/couchbase-linux-x64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linux-x64-napi/-/couchbase-linux-x64-napi-4.2.9.tgz", - "integrity": "sha512-SCo5ZorStwIEXVDcTv23VXWurfrv3JS0hc3sayAP/Rfa+YClMmISAV7J+8V64PeHcfMIWBNCaCof/p/zk/gp0g==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linux-x64-napi/-/couchbase-linux-x64-napi-4.3.0.tgz", + "integrity": "sha512-uxvPU+sOo+x3uQpDRHErfh9suZ4G7iSPtcPCoQGbYPt4+erhPx8Qg5wXTxWYfhcKCh4NCfWten9UwolNgYjIiA==", "cpu": [ "x64" ], @@ -2335,9 +2337,9 @@ } }, "node_modules/@couchbase/couchbase-linuxmusl-x64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linuxmusl-x64-napi/-/couchbase-linuxmusl-x64-napi-4.2.9.tgz", - "integrity": "sha512-d1Ax5SI24yzKFYKuaiazY7GgYLw10zN3i82zZAX2emPikitT0lopPhrDIj8ZXWG4t68+x7IT0qFrvSjpXfyOXQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-linuxmusl-x64-napi/-/couchbase-linuxmusl-x64-napi-4.3.0.tgz", + "integrity": "sha512-f7B4TaVLQi0/X3vDmIF/UZqpTFedtm5Quf9C3wRA3zpWOezsBh5wnkEQ2iWDj9JB67/7uQSmjrWl1ouLDpKkSw==", "cpu": [ "x64" ], @@ -2350,9 +2352,9 @@ } }, "node_modules/@couchbase/couchbase-win32-x64-napi": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-win32-x64-napi/-/couchbase-win32-x64-napi-4.2.9.tgz", - "integrity": "sha512-sg9TKwqDim2GJ3DDztgr4x3EITxBHRX1zKi0VVcURpoRbSBEkrrKLmHFoB7a9nhaJp50W2ZqB60Y97KNTtmUEw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-win32-x64-napi/-/couchbase-win32-x64-napi-4.3.0.tgz", + "integrity": "sha512-acu3+mCmBmwvoUR8YX4vwMadGRo8N5QQOn2qiYIm6Dga5H5wCd5BxpfB3IyDufAqNtpLs/dBULJAhrgyVw+Jow==", "cpu": [ "x64" ], @@ -2607,6 +2609,14 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.4.tgz", + "integrity": "sha512-8zJ8N1x51xo9hwPh6AWnKdLGEC5N3lDa6kms1YHmFBoRhTpJR6HG8wWk0td1MVCu9cD4YBrvjZEtd5Obw0Fbnw==", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2822,6 +2832,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mongodb": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-4.0.7.tgz", + "integrity": "sha512-lPUYPpzA43baXqnd36cZ9xxorprybxXDzteVKCPAdp14ppHtFJHnXYvNpmBvtMUTb5fKXVv6sVbzo1LHkWhJlw==", + "deprecated": "mongodb provides its own types. @types/mongodb is no longer needed.", + "dev": true, + "dependencies": { + "mongodb": "*" + } + }, "node_modules/@types/ms": { "version": "0.7.34", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", @@ -2927,6 +2947,19 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.4.tgz", + "integrity": "sha512-lXCmTWSHJvf0TRSO58nm978b8HJ/EdsSsEKLd3ODHFjo+3VGAyyTp4v50nWvwtzBxSMQrVOK7tcuN0zGPLICMw==", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "4.33.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.33.0.tgz", @@ -3859,6 +3892,14 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/bson": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.4.0.tgz", + "integrity": "sha512-6/gSSEdbkuFlSb+ufj5jUSU4+wo8xQOwm2bDSqwmxiPE17JTpsP63eAwoN8iF8Oy4gJYj+PAL3zdRCTdaw5Y1g==", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/buffer": { "version": "5.7.1", "funding": [ @@ -4842,9 +4883,9 @@ "license": "MIT" }, "node_modules/couchbase": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/couchbase/-/couchbase-4.2.9.tgz", - "integrity": "sha512-NtztuSLyi6GtQbLPBW7uuiRWj6VIh0PcHjtqEAG9nQXyhMkLjiDMGaibxx5rqTud0mai0J8Swif61ENdDRQasw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/couchbase/-/couchbase-4.3.0.tgz", + "integrity": "sha512-wUW9iUH8+frVs0aMiZEso5OEGIFoblmCSmaCU7U/EIqZblSgSNPbvwAE/7QAmAJb8MFjZpEHmMHjPMZXHR2fhw==", "hasInstallScript": true, "dependencies": { "cmake-js": "^7.2.1", @@ -4854,12 +4895,12 @@ "node": ">=16" }, "optionalDependencies": { - "@couchbase/couchbase-darwin-arm64-napi": "4.2.9", - "@couchbase/couchbase-darwin-x64-napi": "4.2.9", - "@couchbase/couchbase-linux-arm64-napi": "4.2.9", - "@couchbase/couchbase-linux-x64-napi": "4.2.9", - "@couchbase/couchbase-linuxmusl-x64-napi": "4.2.9", - "@couchbase/couchbase-win32-x64-napi": "4.2.9" + "@couchbase/couchbase-darwin-arm64-napi": "4.3.0", + "@couchbase/couchbase-darwin-x64-napi": "4.3.0", + "@couchbase/couchbase-linux-arm64-napi": "4.3.0", + "@couchbase/couchbase-linux-x64-napi": "4.3.0", + "@couchbase/couchbase-linuxmusl-x64-napi": "4.3.0", + "@couchbase/couchbase-win32-x64-napi": "4.3.0" } }, "node_modules/couchbase/node_modules/cmake-js": { @@ -7234,7 +7275,7 @@ }, "node_modules/ip": { "version": "2.0.0", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/is-alphabetical": { @@ -8262,6 +8303,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" + }, "node_modules/memory-stream": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/memory-stream/-/memory-stream-0.0.3.tgz", @@ -9172,6 +9218,60 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.43.0.tgz", "integrity": "sha512-cnoqwQi/9fml2Szamv1XbSJieGJ1Dc8tENVMD26Kcfl7xGQWp7OBKMjlwKVGYFJ3/AXJjSOGvcqK7Ry/j9BM1Q==" }, + "node_modules/mongodb": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.4.0.tgz", + "integrity": "sha512-MdFHsyb1a/Ee0H3NmzWTSLqchacDV/APF0H6BNQvraWrOiIocys2EmTFZPgHxWhcfO94c1F34I9MACU7x0hHKA==", + "dependencies": { + "@mongodb-js/saslprep": "^1.1.0", + "bson": "^6.4.0", + "mongodb-connection-string-url": "^3.0.0" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.0.tgz", + "integrity": "sha512-t1Vf+m1I5hC2M5RJx/7AtxgABy1cZmIPQRMXw+gEIPn/cZNF3Oiy+l0UIypUwVB5trcWHq3crg2g3uAR9aAwsQ==", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^13.0.0" + } + }, "node_modules/ms": { "version": "2.1.2", "license": "MIT" @@ -10473,7 +10573,6 @@ }, "node_modules/punycode": { "version": "2.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -11611,7 +11710,7 @@ }, "node_modules/smart-buffer": { "version": "4.2.0", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">= 6.0.0", @@ -11620,7 +11719,7 @@ }, "node_modules/socks": { "version": "2.7.1", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "ip": "^2.0.0", @@ -11679,6 +11778,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/spawn-command": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", @@ -12180,6 +12287,17 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "^2.3.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/traverse": { "version": "0.3.9", "dev": true, @@ -12785,6 +12903,14 @@ "defaults": "^1.0.3" } }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, "node_modules/webpack": { "version": "5.75.0", "dev": true, @@ -12944,6 +13070,18 @@ "node": ">=6" } }, + "node_modules/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==", + "dependencies": { + "tr46": "^4.1.1", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/which": { "version": "2.0.2", "license": "ISC", diff --git a/package.json b/package.json index 62ee9558..4d2ab58f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "vscode-couchbase", "displayName": "Couchbase", "description": "", - "version": "1.2.1", + "version": "1.2.2", "engines": { "vscode": "^1.63.1" }, @@ -48,6 +48,7 @@ "@babel/traverse": "^7.22.8", "@types/glob": "^7.2.0", "@types/mocha": "^8.2.3", + "@types/mongodb": "^4.0.7", "@types/node": "^12.20.42", "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", @@ -109,6 +110,7 @@ "gitly": "^2.4.2", "keytar": "^7.7.0", "monaco-editor": "^0.43.0", + "mongodb": "^6.4.0", "node-addon-api": "^6.0.0", "prop-types": "^15.7.2", "react": "^18.2.0", @@ -541,6 +543,11 @@ "title": "Data Export", "category": "Couchbase" }, + { + "command": "vscode-couchbase.tools.mdbMigrate", + "title": "MongoDB Migrate", + "category": "Couchbase" + }, { "command": "vscode-couchbase.tools.dataImport", "title": "Data Import", @@ -809,6 +816,10 @@ "command": "vscode-couchbase.tools.dataExport", "when": "false" }, + { + "command": "vscode-couchbase.tools.mdbMigrate", + "when": "false" + }, { "command": "vscode-couchbase.tools.dataImport", "when": "false" @@ -993,6 +1004,10 @@ "command": "vscode-couchbase.tools.dataExport", "group": "navigation" }, + { + "command": "vscode-couchbase.tools.mdbMigrate", + "group": "navigation" + }, { "command": "vscode-couchbase.tools.dataImport", "group": "navigation" @@ -1051,4 +1066,4 @@ } ] } -} +} \ No newline at end of file diff --git a/src/commands/extensionCommands/commands.ts b/src/commands/extensionCommands/commands.ts index 456f0a0d..50a7cc57 100644 --- a/src/commands/extensionCommands/commands.ts +++ b/src/commands/extensionCommands/commands.ts @@ -55,6 +55,7 @@ export namespace Commands { export const refreshClusterOverview: string = "vscode-couchbase.refreshClusterOverview"; export const checkAndCreatePrimaryIndex: string = "vscode-couchbase.checkAndCreatePrimaryIndex"; export const dataExport: string = "vscode-couchbase.tools.dataExport"; + export const mdbMigrate: string = "vscode-couchbase.tools.mdbMigrate"; export const dataImport: string = "vscode-couchbase.tools.dataImport"; export const ddlExport: string = "vscode-couchbase.tools.DDLExport"; export const couchbaseIqViewsCommand: string = "couchbase-iq"; diff --git a/src/commands/tools/dataImport.ts b/src/commands/tools/dataImport.ts index 46681bcc..bb3fd959 100644 --- a/src/commands/tools/dataImport.ts +++ b/src/commands/tools/dataImport.ts @@ -1154,15 +1154,15 @@ export class DataImport { } }); } catch (err) { - logger.error(`Failed to open data export webview`); + logger.error(`Failed to open data import webview`); logger.debug(err); vscode.window.showErrorMessage( - "Failed to open data export webview: " + err + "Failed to open data import webview: " + err ); } currentPanel.onDidDispose(() => { - Memory.state.update(Constants.DATA_EXPORT_WEBVIEW, null); + Memory.state.update(Constants.DATA_IMPORT_WEBVIEW, null); }); }; } diff --git a/src/extension.ts b/src/extension.ts index ce542cdb..b6d27351 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -65,6 +65,7 @@ import { clearDocumentFilter } from "./commands/documents/clearDocumentFilter"; import { getClusterOverviewData } from "./util/OverviewClusterUtils/getOverviewClusterData"; import { checkAndCreatePrimaryIndex } from "./commands/indexes/checkAndCreatePrimaryIndex"; import { dataExport } from "./pages/Tools/DataExport/dataExport"; +import { mdbMigrate } from "./pages/Tools/MdbMigrate/mdbMigrate"; import { DataImport } from "./commands/tools/dataImport"; import { ddlExport } from "./commands/tools/ddlExport/ddlExport"; import { CouchbaseIqWebviewProvider } from "./commands/iq/couchbaseIqWebviewProvider"; @@ -121,10 +122,10 @@ export function activate(context: vscode.ExtensionContext) { subscriptions.push( vscode.window.registerWebviewViewProvider( Commands.queryWorkbench, - workbenchWebviewProvider + workbenchWebviewProvider ) ); - + const couchbaseIqWebviewProvider = new CouchbaseIqWebviewProvider(context, cacheService); subscriptions.push( vscode.window.registerWebviewViewProvider( @@ -141,7 +142,7 @@ export function activate(context: vscode.ExtensionContext) { subscriptions.push( vscode.commands.registerCommand( Commands.logoutIq, - ()=>{ + () => { iqLogoutHandler(); } ) @@ -150,7 +151,7 @@ export function activate(context: vscode.ExtensionContext) { subscriptions.push( vscode.commands.registerCommand( Commands.newIqChat, - ()=>{ + () => { newChatHandler(); } ) @@ -159,12 +160,12 @@ export function activate(context: vscode.ExtensionContext) { subscriptions.push( vscode.commands.registerCommand( Commands.showIqSettings, - ()=>{ - vscode.commands.executeCommand('workbench.action.openSettings', "couchbase.iq"); + () => { + vscode.commands.executeCommand('workbench.action.openSettings', "couchbase.iq"); } ) ); - + subscriptions.push( vscode.window.onDidChangeActiveTextEditor(async (editor) => { @@ -512,6 +513,15 @@ export function activate(context: vscode.ExtensionContext) { ) ); + subscriptions.push( + vscode.commands.registerCommand( + Commands.mdbMigrate, + async () => { + await mdbMigrate(context); + } + ) + ); + subscriptions.push( vscode.commands.registerCommand( Commands.dataImport, @@ -647,7 +657,7 @@ export function activate(context: vscode.ExtensionContext) { ); context.subscriptions.push( - vscode.commands.registerCommand(Commands.forceUpdateCache, async ()=>{ + vscode.commands.registerCommand(Commands.forceUpdateCache, async () => { cacheService.fullCache(true); }) ); diff --git a/src/handlers/handleCLIDownloader.ts b/src/handlers/handleCLIDownloader.ts index c4ae3bc6..49f6881f 100644 --- a/src/handlers/handleCLIDownloader.ts +++ b/src/handlers/handleCLIDownloader.ts @@ -19,12 +19,15 @@ class DependenciesDownloader { private readonly TOOL_SHELL = "shell"; private readonly TOOL_IMPORT_EXPORT = "import_export"; private readonly ALL_TOOLS = "all_tools"; + private readonly TOOL_MDB_MIGRATE = "cb_migrate"; private getToolInstallPath(toolKey: string): string { if (toolKey === this.TOOL_SHELL) { return "cbshell"; } else if (toolKey === this.TOOL_IMPORT_EXPORT) { return "cbimport_export"; + } else if (toolKey === this.TOOL_MDB_MIGRATE) { + return "cbmigrate"; } else if (toolKey === this.ALL_TOOLS) { return "cbtools"; } else { @@ -49,6 +52,8 @@ class DependenciesDownloader { CBToolsType.CB_EXPORT, path.join(pathPrefix, "cbexport" + suffix) ); + } else if (toolKey === this.TOOL_MDB_MIGRATE) { + map.set(CBToolsType.CB_MIGRATE, "cbmigrate" + suffix); } else if (toolKey === this.ALL_TOOLS) { map.set( CBToolsType.CBC_PILLOW_FIGHT, @@ -99,6 +104,14 @@ class DependenciesDownloader { OSUtil.MACOS_64 ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_darwin_amd64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.MACOS_64 + ) + ); } else if (os === OSUtil.MACOS_ARM) { map.set( this.TOOL_SHELL, @@ -124,6 +137,14 @@ class DependenciesDownloader { OSUtil.MACOS_ARM ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_darwin_arm64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.MACOS_ARM + ) + ); } else if (os === OSUtil.WINDOWS_64) { map.set( this.TOOL_SHELL, @@ -149,6 +170,14 @@ class DependenciesDownloader { OSUtil.WINDOWS_64 ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_windows_amd64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.WINDOWS_64 + ) + ); } else if (os === OSUtil.WINDOWS_ARM) { map.set( this.TOOL_SHELL, @@ -174,6 +203,14 @@ class DependenciesDownloader { OSUtil.WINDOWS_ARM ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_windows_amd64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.WINDOWS_ARM + ) + ); } else if (os === OSUtil.LINUX_64) { map.set( this.TOOL_SHELL, @@ -199,6 +236,14 @@ class DependenciesDownloader { OSUtil.LINUX_64 ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_linux_amd64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.LINUX_64 + ) + ); } else if (os === OSUtil.LINUX_ARM) { map.set( this.TOOL_SHELL, @@ -216,6 +261,14 @@ class DependenciesDownloader { OSUtil.LINUX_ARM ) ); + map.set( + this.TOOL_MDB_MIGRATE, + this.getToolSpec( + "https://intellij-plugin-dependencies.s3.us-east-2.amazonaws.com/cbmigrate/cbmigrate_0.0.1-beta_linux_arm64.zip", + this.TOOL_MDB_MIGRATE, + OSUtil.LINUX_ARM + ) + ); } else { throw new Error("OS not supported."); } @@ -223,69 +276,21 @@ class DependenciesDownloader { } public handleCLIDownloader = () => { - const extensionPath = path.join(__filename, "..", "cb-vscode-extension"); + const extensionPath = path.join( + __filename, + "..", + "cb-vscode-extension" + ); createFolder(extensionPath); const toolsPath = path.join(extensionPath, "tools"); createFolder(toolsPath); const osArch = OSUtil.getOSArch(); const downloads: Map = this.getDownloadList(osArch); - const shell = downloads.get(this.TOOL_SHELL); - if (shell === undefined) { - return; - } - const shellPath = path.join(toolsPath, shell.getInstallationPath()); - const shellTool = CBTools.getTool(CBToolsType.SHELL); - const shellStatus = shellTool.status; - const toolShellDownloadsMap = downloads.get(this.TOOL_SHELL); - if (toolShellDownloadsMap === undefined) { - return; - } - if ( - shellStatus === ToolStatus.NOT_AVAILABLE && - !this.isInstalled(toolsPath, toolShellDownloadsMap, CBToolsType.SHELL) - ) { - // Avoiding 2 threads to install the same thing at the same time - logger.info("Downloading CB Shell."); - shellTool.status = ToolStatus.DOWNLOADING; - this.downloadAndUnzip(shellPath, shell); - } else { - logger.debug("CBShell is already installed"); - this.setToolActive(ToolStatus.AVAILABLE, shellPath, shell); - } - const cbImport: ToolSpec | undefined = downloads.get( - this.TOOL_IMPORT_EXPORT - ); - if (cbImport === undefined) { - return; - } - const cbImportDir: string = path.join( - toolsPath, - cbImport.getInstallationPath() - ); - const toolImpExportMap = downloads.get(this.TOOL_IMPORT_EXPORT); - if (toolImpExportMap === undefined) { - return; - } - if ( - CBTools.getTool(CBToolsType.CB_IMPORT).status === - ToolStatus.NOT_AVAILABLE && - !this.isInstalled(toolsPath, toolImpExportMap, CBToolsType.CB_EXPORT) - ) { - logger.info( - "Downloading CB Import/Export. The feature will be automatically enabled when the download is complete." - ); - - const cbExportTool = CBTools.getTool(CBToolsType.CB_EXPORT); - const cbImportTool = CBTools.getTool(CBToolsType.CB_IMPORT); - - cbExportTool.status = ToolStatus.DOWNLOADING; - cbImportTool.status = ToolStatus.DOWNLOADING; + // Installs the tools if not already installed + this.manageShellInstallation(downloads, toolsPath); + this.manageCbMigrateInstallation(downloads, toolsPath); + this.manageDataImportExportInstallation(downloads, toolsPath); - this.downloadAndUnzip(cbImportDir, cbImport); - } else { - logger.info("CB Import/Export is already installed"); - this.setToolActive(ToolStatus.AVAILABLE, cbImportDir, cbImport); - } }; private setToolActive( @@ -346,6 +351,112 @@ class DependenciesDownloader { return fs.existsSync(toolPath); } + public manageShellInstallation(downloads: Map, toolsPath: string): void { + const shell = downloads.get(this.TOOL_SHELL); + if (shell === undefined) { + return; + } + const shellPath = path.join(toolsPath, shell.getInstallationPath()); + const shellTool = CBTools.getTool(CBToolsType.SHELL); + const shellStatus = shellTool.status; + const toolShellDownloadsMap = downloads.get(this.TOOL_SHELL); + if (toolShellDownloadsMap === undefined) { + return; + } + if ( + shellStatus === ToolStatus.NOT_AVAILABLE && + !this.isInstalled( + toolsPath, + toolShellDownloadsMap, + CBToolsType.SHELL + ) + ) { + // Avoiding 2 threads to install the same thing at the same time + logger.info("Downloading CB Shell."); + shellTool.status = ToolStatus.DOWNLOADING; + this.downloadAndUnzip(shellPath, shell); + } else { + logger.debug("CBShell is already installed"); + this.setToolActive(ToolStatus.AVAILABLE, shellPath, shell); + } + } + + public manageCbMigrateInstallation(downloads: Map, toolsPath: string): void { + const cbMigrate = downloads.get(this.TOOL_MDB_MIGRATE); + if (cbMigrate === undefined) { + return; + } + const cbMigratePath = path.join( + toolsPath, + cbMigrate.getInstallationPath() + ); + const cbMigrateTool = CBTools.getTool(CBToolsType.CB_MIGRATE); + const cbMigrateStatus = cbMigrateTool.status; + const cbMigrateDownloadsMap = downloads.get(this.TOOL_SHELL); + if (cbMigrateDownloadsMap === undefined) { + return; + } + if ( + cbMigrateStatus === ToolStatus.NOT_AVAILABLE && + !this.isInstalled( + toolsPath, + cbMigrateDownloadsMap, + CBToolsType.CB_MIGRATE + ) + ) { + // Avoiding 2 threads to install the same thing at the same time + logger.info("Downloading CB Migrate."); + cbMigrateTool.status = ToolStatus.DOWNLOADING; + this.downloadAndUnzip(cbMigratePath, cbMigrate); + } else { + logger.debug("CBMigrate is already installed"); + this.setToolActive(ToolStatus.AVAILABLE, cbMigratePath, cbMigrate); + } + } + + public manageDataImportExportInstallation(downloads: Map, toolsPath: string): void { + const cbImport: ToolSpec | undefined = downloads.get( + this.TOOL_IMPORT_EXPORT + ); + if (cbImport === undefined) { + return; + } + const cbImportDir: string = path.join( + toolsPath, + cbImport.getInstallationPath() + ); + const toolImpExportMap = downloads.get(this.TOOL_IMPORT_EXPORT); + if (toolImpExportMap === undefined) { + return; + } + if ( + CBTools.getTool(CBToolsType.CB_IMPORT).status === + ToolStatus.NOT_AVAILABLE && + !this.isInstalled( + toolsPath, + toolImpExportMap, + CBToolsType.CB_EXPORT + ) + ) { + logger.info( + "Downloading CB Import/Export. The feature will be automatically enabled when the download is complete." + ); + + const cbExportTool = CBTools.getTool(CBToolsType.CB_EXPORT); + const cbImportTool = CBTools.getTool(CBToolsType.CB_IMPORT); + + cbExportTool.status = ToolStatus.DOWNLOADING; + cbImportTool.status = ToolStatus.DOWNLOADING; + + this.downloadAndUnzip(cbImportDir, cbImport); + } else { + logger.info("CB Import/Export is already installed"); + this.setToolActive(ToolStatus.AVAILABLE, cbImportDir, cbImport); + } + } + + + // This is a test function, keeping this here to check for any specific downloaded CLI tool /* public runFile(targetDir:string) { const scriptPath = path.join(targetDir, 'cbsh'); diff --git a/src/pages/Tools/MdbMigrate/mdbMigrate.ts b/src/pages/Tools/MdbMigrate/mdbMigrate.ts new file mode 100644 index 00000000..2838e935 --- /dev/null +++ b/src/pages/Tools/MdbMigrate/mdbMigrate.ts @@ -0,0 +1,258 @@ +import { MDBToCB } from "../../../tools/MDBMigrate"; +import { + CBTools, + Type as CBToolsType, +} from "../../../util/DependencyDownloaderUtils/CBTool"; +import * as vscode from "vscode"; +import { getActiveConnection } from "../../../util/connections"; +import { Memory } from "../../../util/util"; +import { Constants } from "../../../util/constants"; +import { logger } from "../../../logger/logger"; +import * as path from "path"; +import { getLoader } from "../../../webViews/loader.webview"; +import { MdbMigrateWebview } from "../../../webViews/tools/mdbMigrate.webview"; +import { IConnection } from "../../../types/IConnection"; +import { MongoClient } from "mongodb"; + +export const getScopes = async (bucketId: string, connection: IConnection) => { + const scopes = await connection.cluster + ?.bucket(bucketId) + .collections() + .getAllScopes(); + return scopes; +}; + +const validateFormData = (formData: any): string => { + const errors = []; + if (!formData.connectionString.trim()) { + errors.push("Please specify a valid Mongodb Connection URI"); + } + + if (!formData.database) { + errors.push("Please specify the Source Database to migrate from"); + } + + if (!formData.collections || formData.collections.length === 0) { + errors.push("Please specify the Source Collections field"); + } + + if (!formData.bucket) { + errors.push("Please select a Couchbase bucket"); + } + + if (!formData.cbScope || formData.cbScope.length === 0) { + errors.push("Please select a Couchbase scope field"); + } + + if (errors.length > 0) { + return errors.join("
"); + } + + return ""; +}; + +export interface IDataMigrateWebviewState { + webviewPanel: vscode.WebviewPanel; +} + +export const mdbMigrate = async (context: vscode.ExtensionContext) => { + const connection = getActiveConnection(); + if (!connection) { + return; + } + if (!CBTools.getTool(CBToolsType.CB_MIGRATE).isAvailable()) { + vscode.window.showErrorMessage( + "CB Migrate is still loading, Please try again later" + ); + return; + } + + const dataMigratetWebviewDetails = + Memory.state.get( + Constants.DATA_MIGRATE_MDB_WEBVIEW + ); + if (dataMigratetWebviewDetails) { + // data migrate webview already exists, Closing existing and creating new + try { + dataMigratetWebviewDetails.webviewPanel.dispose(); + } catch (e) { + logger.error("Error while disposing data migrate webview: " + e); + } + Memory.state.update(Constants.DATA_MIGRATE_MDB_WEBVIEW, null); + } + + const currentPanel = vscode.window.createWebviewPanel( + "dataMigrate", + "Data Migration from MDB to Couchbase", + vscode.ViewColumn.One, + { + enableScripts: true, + enableForms: true, + } + ); + Memory.state.update(Constants.DATA_MIGRATE_MDB_WEBVIEW, { + webviewPanel: currentPanel, + }); + currentPanel.iconPath = { + dark: vscode.Uri.file( + path.join( + __filename, + "..", + "..", + "images", + "dark", + "export_dark.svg" + ) + ), + light: vscode.Uri.file( + path.join( + __filename, + "..", + "..", + "images", + "light", + "export_light.svg" + ) + ), + }; + currentPanel.webview.html = getLoader("Data Migrate"); + + // Get all buckets + const buckets = await connection.cluster?.buckets().getAllBuckets(); + if (buckets === undefined) { + vscode.window.showErrorMessage("Buckets not found"); + return; + } + + const bucketNameArr: string[] = buckets.map((b) => b.name); + + async function connectToMongoDB( + connectionString: string + ): Promise { + try { + const client: MongoClient = await MongoClient.connect( + connectionString + ); + logger.info("Connected successfully to MongoDB"); + const adminDb = client.db().admin(); + logger.info("fetching list of databases"); + const databasesList = await adminDb.listDatabases(); + const databases = databasesList.databases.map((db) => db.name); + await client.close(); + + return databases; + } catch (error) { + const error_message = + "Unable to Connect to MongoDB, Please check Connection URI and try again"; + console.error("Error while connecting to MongoDB:", error); + currentPanel.webview.postMessage({ + command: "vscode-couchbase.tools.mdbMigrate.formValidationError", + error: error_message, + }); + throw new Error("Failed to connect to MongoDB: " + error); + } + } + + async function getCollectionsList( + connectionString: string, + databaseName: string + ): Promise { + try { + const client: MongoClient = await MongoClient.connect( + connectionString + ); + // Connect to the specific database + const db = client.db(databaseName); + + // List collections in the specific database + const collections = await db.listCollections().toArray(); + const collectionNames: string[] = collections.map( + (collection) => collection.name + ); + await client.close(); + + return collectionNames; + } catch (error) { + console.error("Error while fetching collections:", error); + throw error; + } + } + + try { + currentPanel.webview.html = await MdbMigrateWebview(bucketNameArr); + currentPanel.webview.onDidReceiveMessage(async (message) => { + switch (message.command) { + // ADD cases here :) + case "vscode-couchbase.tools.mdbMigrate.Migrate": + const formData = message.data; + const validationError = validateFormData(formData); + if (validationError === "") { + // There is no error + MDBToCB.export( + formData.connectionString, + formData.database, + formData.collections, + formData.indexes, + formData.bucket, + formData.cbScope, + context + ); + } else { + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.mdbMigrate.formValidationError", + error: validationError, + }); + } + break; + case "vscode-couchbase.tools.mdbMigrate.getCollections": + const collection = await getCollectionsList( + message.connectionString, + message.databases + ); + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.mdbMigrate.collectionInfo", + collection: collection, + }); + + break; + + case "vscode-couchbase.tools.mdbMigrate.getCbScopes": + const scopes = await getScopes( + message.bucketId, + connection + ); + if (scopes === undefined) { + vscode.window.showErrorMessage("Scopes are undefined"); + break; + } + currentPanel.webview.postMessage({ + command: "vscode-couchbase.tools.mdbMigrate.scopesInfo", + scopes: scopes, + }); + break; + + case "vscode-couchbase.tools.mdbMigrate.getDatabases": + const databases = await connectToMongoDB( + message.connectionString + ); + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.mdbMigrate.databaseInfo", + databases: databases, + }); + } + }); + } catch (err) { + logger.error(`Failed to open data migrate webview`); + logger.debug(err); + vscode.window.showErrorMessage( + "Failed to open data migrate webview: " + err + ); + } + + currentPanel.onDidDispose(() => { + Memory.state.update(Constants.DATA_MIGRATE_MDB_WEBVIEW, null); + }); +}; diff --git a/src/tools/MDBMigrate.ts b/src/tools/MDBMigrate.ts new file mode 100644 index 00000000..91fa2c2a --- /dev/null +++ b/src/tools/MDBMigrate.ts @@ -0,0 +1,74 @@ +import { CBTools, Type } from "../util/DependencyDownloaderUtils/CBTool"; +import { getActiveConnection, getConnectionId } from "../util/connections"; +import { Constants } from "../util/constants"; +import * as keytar from "keytar"; +import * as vscode from "vscode"; + +export class MDBToCB { + static async export( + mDBConnectionString: string, + databases: string, + collections: string[], + indexes: string, + cbBucket: string, + cbScope: string, + context: vscode.ExtensionContext + ): Promise { + const connection = getActiveConnection(); + if (!connection) { + return; + } + + const password = await keytar.getPassword( + Constants.extensionID, + getConnectionId(connection) + ); + if (!password) { + return undefined; + } + try { + // Traverse the collections to build seperate commands + for (const collection of collections) { + // Build Command + const cmd: string[] = []; + cmd.push(CBTools.getTool(Type.CB_MIGRATE).path); + cmd.push("mongo "); + cmd.push("--mongodb-uri"); + cmd.push(mDBConnectionString); + cmd.push("--mongodb-database"); + cmd.push(databases); + cmd.push("--mongodb-collection"); + cmd.push(collection); + cmd.push("--cb-cluster"); + cmd.push(connection.url); + if (indexes) { + cmd.push("--copy-indexes"); + } + cmd.push("--cb-username"); + cmd.push(connection.username); + cmd.push("--cb-password"); + cmd.push(password); + cmd.push("--cb-bucket"); + cmd.push(cbBucket); + cmd.push("--cb-scope"); + cmd.push(cbScope); + cmd.push("--cb-generate-key"); + cmd.push("%_id%"); + + cmd.push("; \n"); + + // Run Command + const terminal: vscode.Terminal = + vscode.window.createTerminal("MDBToCb"); + const text = cmd.join(" "); + terminal.sendText(text); + terminal.show(); + } + } catch (error) { + console.error( + "An error occurred while trying to migrate the databases from mongodb to couchbase" + ); + console.error(error); + } + } +} diff --git a/src/util/DependencyDownloaderUtils/CBTool.ts b/src/util/DependencyDownloaderUtils/CBTool.ts index 9e4485f0..bed06f7d 100644 --- a/src/util/DependencyDownloaderUtils/CBTool.ts +++ b/src/util/DependencyDownloaderUtils/CBTool.ts @@ -19,7 +19,8 @@ export enum Type { CB_IMPORT = "CB_IMPORT", CB_EXPORT = "CB_EXPORT", CBC_PILLOW_FIGHT = "CBC_PILLOW_FIGHT", - MCTIMINGS = "MCTIMINGS" + MCTIMINGS = "MCTIMINGS", + CB_MIGRATE = "CB_MIGRATE" } diff --git a/src/util/constants.ts b/src/util/constants.ts index 16229d84..f234c697 100644 --- a/src/util/constants.ts +++ b/src/util/constants.ts @@ -100,5 +100,6 @@ export class Constants { public static CURRENT_VBUCKET_REPLICA_ITEMS = "Current vBucket Replica Items"; public static QUERY_RESULT = "Couchbase Query Result"; public static DATA_EXPORT_WEBVIEW = "dataExportWebview"; + public static DATA_MIGRATE_MDB_WEBVIEW = "dataMdbMigrateWebview"; public static DATA_IMPORT_WEBVIEW = "dataImportWebview"; } diff --git a/src/webViews/tools/mdbMigrate.webview.ts b/src/webViews/tools/mdbMigrate.webview.ts new file mode 100644 index 00000000..047d439a --- /dev/null +++ b/src/webViews/tools/mdbMigrate.webview.ts @@ -0,0 +1,519 @@ +export const MdbMigrateWebview = async (buckets: string[]): Promise => { + return /*html*/` + + + + + + Data Migrate + + + + + +

Mongodb to Couchbase Data Migration

+
+

This tool utilizes the CLI in the background to migrate the data.

+

Please refer to this for more + information on how to use the CLI.

+
+ Source +
+
+ +
+ +
+ Connect
+
+
+ + +
+ + +
+
+ + + Check to include indexes in the migration. Indexes are included by default. +
+
+ Target +
+
+ + +
+ + +
+
+ +
+ + + + + + `; +}; +