From 7cdffdceeecd21695ec55c724e1584e11ed065f1 Mon Sep 17 00:00:00 2001 From: Lokesh Goel <113521973+lokesh-couchbase@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:29:59 +0530 Subject: [PATCH] DA-316 Add CB Import Functionality (#318) --- .prettierrc | 4 + images/dark/database-import.svg | 7 + images/light/database-import.svg | 7 + package-lock.json | 60 +- package.json | 22 +- src/commands/extensionCommands/commands.ts | 1 + src/commands/tools/dataImport.ts | 990 ++++++++++++++++++ src/extension.ts | 10 + src/pages/Tools/DataExport/dataExport.ts | 4 +- src/pages/overviewCluster/overviewCluster.ts | 1 - src/tools/CBExport.ts | 4 +- src/tools/CBImport.ts | 142 +++ src/util/constants.ts | 1 + src/webViews/favoriteQueries.webiew.ts | 6 +- src/webViews/tools/dataExport.webview.ts | 39 +- .../getDatasetAndCollection.webview.ts | 381 +++++++ .../getKeysAndAdvancedSettings.webview.ts | 406 +++++++ .../tools/dataImport/getSummary.webview.ts | 275 +++++ 18 files changed, 2316 insertions(+), 44 deletions(-) create mode 100644 .prettierrc create mode 100644 images/dark/database-import.svg create mode 100644 images/light/database-import.svg create mode 100644 src/commands/tools/dataImport.ts create mode 100644 src/tools/CBImport.ts create mode 100644 src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts create mode 100644 src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts create mode 100644 src/webViews/tools/dataImport/getSummary.webview.ts diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..5a938ce1 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "useTabs": false +} diff --git a/images/dark/database-import.svg b/images/dark/database-import.svg new file mode 100644 index 00000000..ac96fa1f --- /dev/null +++ b/images/dark/database-import.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/images/light/database-import.svg b/images/light/database-import.svg new file mode 100644 index 00000000..0b2956bb --- /dev/null +++ b/images/light/database-import.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 821fb305..1ef0767f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,13 @@ "@monaco-editor/react": "^4.5.2", "@popperjs/core": "^2.11.8", "@types/decompress": "^4.2.4", + "@types/stream-json": "^1.7.5", "ag-grid-community": "^29.3.5", "ag-grid-react": "^29.3.5", "axios": "^1.4.0", "clsx": "^2.0.0", "couchbase": "^4.2.1", + "csv-parse": "^5.5.2", "d3-hierarchy": "^1.1.9", "d3-selection": "^1.4.2", "d3-shape": "^1.3.7", @@ -33,7 +35,9 @@ "react-merge-refs": "^2.0.2", "react-popper": "^2.3.0", "react-popper-tooltip": "^4.4.2", - "react-router-dom": "^6.14.2" + "react-router-dom": "^6.14.2", + "stream-json": "^1.8.0", + "uuid": "^9.0.1" }, "devDependencies": { "@babel/parser": "^7.22.7", @@ -44,6 +48,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "@types/tar": "^6.1.5", + "@types/uuid": "^9.0.6", "@types/vscode": "^1.63.1", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", @@ -743,6 +748,23 @@ "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", "dev": true }, + "node_modules/@types/stream-chain": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.3.tgz", + "integrity": "sha512-cwWE6mrdDpmW3B5wr1+vpjbg8h3hZfOr/PbKQ38VE21xNyF64GNjrh855YdNAPCt4kSYXuLwgRqBWkY/dD6KMg==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stream-json": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.5.tgz", + "integrity": "sha512-IVTtojYNqc6RT9FWBlwPLG6QTVdv2gHdqHOyBYPgcsCKfwIxcQpKw0e0ybQVyCvJJT9Le6w9RB5r5c6mo2m+IQ==", + "dependencies": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "node_modules/@types/tar": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.5.tgz", @@ -762,6 +784,12 @@ "node": ">=8" } }, + "node_modules/@types/uuid": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.6.tgz", + "integrity": "sha512-BT2Krtx4xaO6iwzwMFUYvWBWkV2pr37zD68Vmp1CDV196MzczBRxuEpD6Pr395HAgebC/co7hOphs53r8V7jew==", + "dev": true + }, "node_modules/@types/vscode": { "version": "1.74.0", "dev": true, @@ -2510,6 +2538,11 @@ "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", "dev": true }, + "node_modules/csv-parse": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.2.tgz", + "integrity": "sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==" + }, "node_modules/d3-color": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", @@ -8143,6 +8176,19 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==" + }, + "node_modules/stream-json": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.8.0.tgz", + "integrity": "sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", @@ -8835,6 +8881,18 @@ "version": "1.0.2", "license": "MIT" }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "dev": true, diff --git a/package.json b/package.json index 2b961999..87031c68 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "@types/tar": "^6.1.5", + "@types/uuid": "^9.0.6", "@types/vscode": "^1.63.1", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", @@ -82,11 +83,13 @@ "@monaco-editor/react": "^4.5.2", "@popperjs/core": "^2.11.8", "@types/decompress": "^4.2.4", + "@types/stream-json": "^1.7.5", "ag-grid-community": "^29.3.5", "ag-grid-react": "^29.3.5", "axios": "^1.4.0", "clsx": "^2.0.0", "couchbase": "^4.2.1", + "csv-parse": "^5.5.2", "d3-hierarchy": "^1.1.9", "d3-selection": "^1.4.2", "d3-shape": "^1.3.7", @@ -103,7 +106,9 @@ "react-merge-refs": "^2.0.2", "react-popper": "^2.3.0", "react-popper-tooltip": "^4.4.2", - "react-router-dom": "^6.14.2" + "react-router-dom": "^6.14.2", + "stream-json": "^1.8.0", + "uuid": "^9.0.1" }, "activationEvents": [ "onFileSystem:couchbase", @@ -522,6 +527,11 @@ "title": "Data Export", "category": "Couchbase" }, + { + "command": "vscode-couchbase.tools.dataImport", + "title": "Data Import", + "category": "Couchbase" + }, { "title": "Query Context", "command": "vscode-couchbase.queryContext", @@ -761,6 +771,10 @@ "command": "vscode-couchbase.tools.dataExport", "when": "false" }, + { + "command": "vscode-couchbase.tools.dataImport", + "when": "false" + }, { "command": "vscode-couchbase.tools.DDLExport", "when": "false" @@ -914,6 +928,10 @@ "command": "vscode-couchbase.tools.dataExport", "group": "navigation" }, + { + "command": "vscode-couchbase.tools.dataImport", + "group": "navigation" + }, { "command": "vscode-couchbase.tools.DDLExport", "group": "navigation" @@ -949,4 +967,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/commands/extensionCommands/commands.ts b/src/commands/extensionCommands/commands.ts index 09dea4f4..b84d31c2 100644 --- a/src/commands/extensionCommands/commands.ts +++ b/src/commands/extensionCommands/commands.ts @@ -55,5 +55,6 @@ export namespace Commands { export const refreshClusterOverview: string = "vscode-couchbase.refreshClusterOverview"; export const checkAndCreatePrimaryIndex: string = "vscode-couchbase.checkAndCreatePrimaryIndex"; export const dataExport: string = "vscode-couchbase.tools.dataExport"; + export const dataImport: string = "vscode-couchbase.tools.dataImport"; export const ddlExport: string = "vscode-couchbase.tools.DDLExport"; } diff --git a/src/commands/tools/dataImport.ts b/src/commands/tools/dataImport.ts new file mode 100644 index 00000000..4298e0a1 --- /dev/null +++ b/src/commands/tools/dataImport.ts @@ -0,0 +1,990 @@ +import { + CBTools, + Type as CBToolsType, +} from "../../util/DependencyDownloaderUtils/CBTool"; +import { getActiveConnection } from "../../util/connections"; +import * as vscode from "vscode"; +import * as path from "path"; +import { Memory } from "../../util/util"; +import { Constants } from "../../util/constants"; +import { logger } from "../../logger/logger"; +import { getLoader } from "../../webViews/loader.webview"; +import { getDatasetAndCollection } from "../../webViews/tools/dataImport/getDatasetAndCollection.webview"; +import * as fs from "fs"; +import { getScopes } from "../../pages/Tools/DataExport/dataExport"; +import { getKeysAndAdvancedSettings } from "../../webViews/tools/dataImport/getKeysAndAdvancedSettings.webview"; +import { CBImport } from "../../tools/CBImport"; +import { parser } from "stream-json"; +import { pick } from "stream-json/filters/Pick"; +import { streamValues } from "stream-json/streamers/StreamValues"; +import * as csv from "csv-parse"; +import { v4 as uuidv4 } from "uuid"; +import { getSummary } from "../../webViews/tools/dataImport/getSummary.webview"; + +interface IDataImportWebviewState { + webviewPanel: vscode.WebviewPanel; +} +export class DataImport { + cachedJsonDocs: string[] = []; + cachedCsvDocs: Map = new Map(); + PREVIEW_SIZE: number = 6; + JSON_FILE_EXTENSION: string = ".json"; + CSV_FILE_EXTENSION: string = ".csv"; + JSON_FILE_FORMAT: string = "json"; + CSV_FILE_FORMAT: string = "csv"; + datasetField: string = ""; + format: string = ""; + readonly UUID_FLAG = "#UUID#"; + readonly MONO_INCR_FLAG = "#MONO_INCR#"; + readonly WORDS_WITH_PERCENT_SYMBOLS_REGEX = "%(\\w+)%"; + + protected fileFormat: string = ""; + constructor() {} + + // Functions to fetch key previews + protected cleanString(input: string): string { + const firstOpenBracket = input.indexOf("{"); + const lastCloseBracket = input.lastIndexOf("}"); + + if ( + firstOpenBracket === -1 || + lastCloseBracket === -1 || + firstOpenBracket > lastCloseBracket + ) { + return input; + } + + return input.substring(firstOpenBracket, lastCloseBracket + 1); + } + + protected isValidBrackets(s: string): boolean { + const stack: string[] = []; + + if (!s.includes("{") || !s.includes("}")) { + return false; + } + + for (const c of s) { + if (c === "{") { + stack.push(c); + } else if ( + c === "}" && + (stack.length === 0 || stack.pop() !== "{") + ) { + return false; + } + } + + return stack.length === 0; + } + + private readAndProcessPartialDataFromDataset = async () => { + try { + const datasetPath: string = this.datasetField; + if (datasetPath.endsWith(this.JSON_FILE_EXTENSION)) { + const readStream = fs.createReadStream(datasetPath, { + encoding: "utf8", + }); + let counter = 0; + let documentFound = ""; + let insideArray = false; + + readStream.on("data", (chunk: string) => { + const lines = chunk.split("\n"); + + for (let line of lines) { + if (counter === 0 && !insideArray) { + if (!line.trim().startsWith("[")) { + logger.debug("Not a JSON array"); + readStream.close(); + return; + } + + insideArray = true; + line = line.replace("[", ""); + } + + if ( + counter > 2000 || + this.cachedJsonDocs.length >= this.PREVIEW_SIZE + ) { + readStream.close(); + return; + } + + counter++; + documentFound += line.trim(); + + if (this.isValidBrackets(documentFound)) { + documentFound = this.cleanString(documentFound); + this.cachedJsonDocs.push(documentFound); + + documentFound = ""; + } + } + }); + + readStream.on("end", () => { + readStream.close(); + }); + } else { + const headers = await this.sampleElementFromCsvFile( + datasetPath, + 1 + ); + if (headers === null) { + return; + } + for ( + let lineNumber = 2; + lineNumber < 2 + this.PREVIEW_SIZE; + lineNumber++ + ) { + const data = await this.sampleElementFromCsvFile( + datasetPath, + lineNumber + ); + if (data === null) { + continue; + } + + for ( + let headersIndex = 0; + headersIndex < headers.length; + headersIndex++ + ) { + let cachedCsv = this.cachedCsvDocs.get( + headers[headersIndex] + ); + if (!cachedCsv) { + cachedCsv = new Array(this.PREVIEW_SIZE); + } + cachedCsv[lineNumber - 2] = data[headersIndex]; + this.cachedCsvDocs.set( + headers[headersIndex], + cachedCsv + ); + } + } + } + } catch (err) { + logger.error(err); + } + }; + + async updateKeyPreview(keyType: string, keyExpr: string): Promise { + if ( + (this.fileFormat === this.JSON_FILE_FORMAT && + this.cachedJsonDocs.length === 0) || + (this.fileFormat === this.CSV_FILE_FORMAT && + this.cachedCsvDocs.size === 0) + ) { + await this.readAndProcessPartialDataFromDataset(); + // Wait 500ms extra to cache everything + await new Promise((resolve) => { + setTimeout(resolve, 500); + }); + } + + const previewContent: string[] = []; + let monoIncrValue = 1; + + if (keyType === "fieldValue") { + const fieldName = keyExpr; + if (this.fileFormat === this.JSON_FILE_FORMAT) { + for ( + let i = 0; + i < Math.min(this.cachedJsonDocs.length, this.PREVIEW_SIZE); + i++ + ) { + const jsonObject = JSON.parse(this.cachedJsonDocs[i]); + if (jsonObject.hasOwnProperty(fieldName)) { + previewContent.push(jsonObject[fieldName].toString()); + } + } + } else if ( + this.fileFormat === this.CSV_FILE_FORMAT && + this.cachedCsvDocs.get(fieldName) !== null + ) { + for ( + let i = 0; + i < Math.min(this.cachedCsvDocs.size, this.PREVIEW_SIZE); + i++ + ) { + let currentContext = this.cachedCsvDocs.get(fieldName); + if (currentContext) { + previewContent.push(currentContext[i].toString()); + } + } + } + } else if (keyType === "customExpression") { + if (this.fileFormat === this.JSON_FILE_FORMAT) { + const expression = keyExpr; + const pattern = new RegExp( + this.WORDS_WITH_PERCENT_SYMBOLS_REGEX, + "g" + ); + let matches: string[] = []; + + let match; + + while ((match = pattern.exec(expression)) !== null) { + // match[1] contains the captured word (the part between % symbols) + if (match[1] === "") { + break; + } + matches.push(match[1]); + } + const fieldNamesList: string[] = []; + matches.forEach((match) => { + fieldNamesList.push(match.replace(/%/g, "")); + }); + + for ( + let i = 0; + i < Math.min(this.cachedJsonDocs.length, this.PREVIEW_SIZE); + i++ + ) { + const jsonObject = JSON.parse(this.cachedJsonDocs[i]); + let keyBuilder = expression; + + fieldNamesList.forEach((fieldName) => { + if (jsonObject.hasOwnProperty(fieldName)) { + keyBuilder = keyBuilder.replace( + new RegExp(`%${fieldName}%`, "g"), + jsonObject[fieldName].toString() + ); + } + }); + + keyBuilder = keyBuilder.replace( + new RegExp(this.UUID_FLAG, "g"), + uuidv4() + ); + keyBuilder = keyBuilder.replace( + new RegExp(this.MONO_INCR_FLAG, "g"), + monoIncrValue.toString() + ); + monoIncrValue++; + + previewContent.push(keyBuilder); + } + } else if (this.fileFormat === this.CSV_FILE_FORMAT) { + const expression = keyExpr; + const pattern = new RegExp( + this.WORDS_WITH_PERCENT_SYMBOLS_REGEX, + "g" + ); + let matches: string[] = []; + + let match; + + while ((match = pattern.exec(expression)) !== null) { + // match[1] contains the captured word (the part between % symbols) + if (match[1] === "") { + break; + } + matches.push(match[1]); + } + const fieldNamesList: string[] = []; + matches.forEach((match) => { + fieldNamesList.push(match.replace(/%/g, "")); + }); + + for ( + let i = 0; + i < Math.min(this.cachedCsvDocs.size, this.PREVIEW_SIZE); + i++ + ) { + let keyBuilder = expression; + + fieldNamesList.forEach((fieldName) => { + const keyContent = this.cachedCsvDocs.get(fieldName); + if (keyContent) { + keyBuilder = keyBuilder.replace( + new RegExp(`%${fieldName}%`, "g"), + keyContent[i].toString() + ); + } + }); + + keyBuilder = keyBuilder.replace( + new RegExp(this.UUID_FLAG, "g"), + uuidv4() + ); + keyBuilder = keyBuilder.replace( + new RegExp(this.MONO_INCR_FLAG, "g"), + monoIncrValue.toString() + ); + monoIncrValue++; + + previewContent.push(keyBuilder); + } + } + } else { + // Random UUID case + for (let i = 0; i < this.PREVIEW_SIZE; i++) { + previewContent.push(uuidv4()); // Shows Random UUID, May not represent real values + } + } + return previewContent.join("\n"); + } + + // Functions to detect validity of scopes and collections + + async sampleElementFromJsonArrayFile( + filePath: string + ): Promise { + return new Promise((resolve, reject) => { + const pipeline = fs + .createReadStream(filePath) + .pipe(parser()) + .pipe(pick({ filter: "0" })) + .pipe(streamValues()); + + let result: string | null = null; + + pipeline.on("data", (data) => { + if (result === null) { + result = JSON.stringify(data.value); + pipeline.destroy(); + } + }); + + pipeline.on("close", () => { + resolve(result); + }); + pipeline.on("error", (err) => { + logger.error(err); + resolve(null); + }); + }); + } + + async sampleElementFromCsvFile( + filePath: string, + lineNumber: number + ): Promise { + return new Promise((resolve, reject) => { + let currentLine = 0; + let result: string[] | null = null; + fs.createReadStream(filePath) + .pipe(csv.parse()) + .on("data", (row) => { + currentLine++; + if (currentLine === lineNumber) { + result = Object.values(row); + resolve(result); + } + }) + .on("end", () => { + if (result === null) { + resolve([]); + } + }) + .on("error", (err) => { + logger.error(err); + resolve([]); + }); + }); + } + + checkFields = async ( + filePath: string, + fieldText: string + ): Promise => { + const pattern = /%(.*?)%/g; + let match; + + while ((match = pattern.exec(fieldText)) !== null) { + const fieldName = match[1]; + try { + if (this.fileFormat === "json") { + const sampleElement = + await this.sampleElementFromJsonArrayFile(filePath); + + if (sampleElement !== null) { + const jsonObject = JSON.parse(sampleElement); + if (!jsonObject.hasOwnProperty(fieldName)) { + return false; + } + } else { + return false; + } + } else if (this.fileFormat === "csv") { + const headers = await this.sampleElementFromCsvFile( + filePath, + 1 + ); + if (headers !== null && !headers.includes(fieldName)) { + return false; + } + } + } catch (e) { + logger.error(e); + return false; + } + } + + return true; + }; + + // Functions to detect Validity of dataset + detectDatasetFormat = async (filePath: string): Promise => { + try { + let currentPosition = 0; + const startBuffer = await this.readFirstTwoNonEmptyCharacters( + filePath + ); + const endBuffer = await this.readLastTwoNonEmptyCharacters( + filePath + ); + + const firstChar = startBuffer[0]; + const secondChar = startBuffer[1]; + + const secondLastChar = endBuffer[0]; + const lastChar = endBuffer[1]; + + if ( + firstChar === "[" && + secondChar === "{" && + secondLastChar === "}" && + lastChar === "]" + ) { + return "list"; + } else if (firstChar === "{" && lastChar === "}") { + return "lines"; + } + return null; + } catch (err) { + throw new Error("" + err); + } + }; + + readLastTwoNonEmptyCharacters = async ( + filePath: string + ): Promise => { + const chunkSize = 1024; // Adjust the chunk size as needed + let buffer = Buffer.alloc(2); // Initialize a buffer to store the last two non-empty characters + let bytesRead = 0; + let lastTwoNonEmptyChars = ""; + + const fd = await fs.promises.open(filePath, "r"); + const fileSize = (await fd.stat()).size; + + for ( + let position = fileSize - 1; + position >= 0 && bytesRead < 2; + position -= chunkSize + ) { + const bufferSize = Math.min(chunkSize, position + 1); + const chunk = Buffer.alloc(bufferSize); + + await fd.read(chunk, 0, bufferSize, position); + + for (let i = bufferSize - 1; i >= 0; i--) { + const char = chunk.toString("utf8", i, i + 1); + if (!/[\s\t\n]/.test(char) && char.charCodeAt(0) !== 0) { + // If the character is not a space, tab, or newline, add it to the buffer + buffer[1] = buffer[0]; // Shift the previous character + buffer[0] = chunk[i]; // Store the current character + + bytesRead++; + lastTwoNonEmptyChars = buffer.toString( + "utf8", + 0, + bytesRead + ); + } + + if (bytesRead >= 2) { + break; + } + } + } + + await fd.close(); + + return lastTwoNonEmptyChars; + }; + + readFirstTwoNonEmptyCharacters = async ( + filePath: string + ): Promise => { + const chunkSize = 1024; // Adjust the chunk size as needed + let buffer = Buffer.alloc(2); // Initialize a buffer to store the first two non-empty characters + let bytesRead = 0; + + const readStream = fs.createReadStream(filePath, { + highWaterMark: chunkSize, + }); + let closed = false; + for await (const chunk of readStream) { + for (let i = 0; i < chunk.length; i++) { + const char = chunk.toString("utf8", i, i + 1); + + if (!/[\s\t\n]/.test(char)) { + // If the character is not a space, tab, or newline, add it to the buffer + buffer[bytesRead] = chunk[i]; // Store the current character + bytesRead++; + + if (bytesRead >= 2) { + // We have found the first two non-empty characters, so close the stream + readStream.close(); + closed = true; + break; + } + } + } + if (closed) { + break; + } + } + return buffer.toString("utf8", 0, bytesRead); + }; + + validateDataset = async (datasetFilePath: string): Promise => { + let errors = []; + // Check if dataset is not empty + if (!datasetFilePath || datasetFilePath.trim() === "") { + errors.push("Dataset is required."); + return errors; + } + logger.debug("Dataset file path received: " + datasetFilePath); + + if (datasetFilePath.endsWith(this.JSON_FILE_EXTENSION)) { + this.fileFormat = this.JSON_FILE_FORMAT; + } else if (datasetFilePath.endsWith(this.CSV_FILE_EXTENSION)) { + this.fileFormat = this.CSV_FILE_FORMAT; + } else { + errors.push("Please enter valid json or csv file only"); + } + + if (this.fileFormat === this.JSON_FILE_FORMAT) { + let currentFormat: string | null = await this.detectDatasetFormat( + datasetFilePath + ); + + if (currentFormat) { + this.format = currentFormat; + logger.info("detected format " + currentFormat); + } else { + this.format = ""; + logger.error("format not detected"); + errors.push("Please enter valid json file format only"); + } + + // Check if given JSON File is correct + } + return errors; + }; + + validateDatasetAndCollectionFormData = async ( + formData: any + ): Promise => { + let errors: string[] = []; + + // Validate Dataset + const datasetFilePath: string = formData.dataset; + const datasetErrors = await this.validateDataset(datasetFilePath); + for (let error of datasetErrors) { + errors.push(error); + } + if (datasetErrors.length === 0) { + this.datasetField = datasetFilePath; + } + + // Check if bucket is not empty + if (!formData.bucket) { + errors.push("Bucket is required."); + } + + // Perform different validation checks based on the value of scopesAndCollections + switch (formData.scopesAndCollections) { + case "SpecifiedCollection": + // Check if scopesDropdown and collectionsDropdown are not empty + if ( + !formData.scopesDropdown || + formData.scopesDropdown === "" + ) { + errors.push("Scope is required for Specified Collection."); + } + if ( + !formData.collectionsDropdown || + formData.collectionsDropdown === "" + ) { + errors.push( + "Collection is required for Specified Collection." + ); + } + break; + case "dynamicCollection": + // Check if scopesDynamicField and collectionsDynamicField are not empty + if ( + !formData.scopesDynamicField || + formData.scopesDynamicField.trim() === "" + ) { + errors.push( + "Scope Field is required for Dynamic Collection." + ); + } else { + const checkFieldsResult = await this.checkFields( + datasetFilePath, + formData.scopesDynamicField + ); + if (!checkFieldsResult) { + errors.push("Scope's field is not valid"); + } + } + if ( + !formData.collectionsDynamicField || + formData.collectionsDynamicField.trim() === "" + ) { + errors.push( + "Collection Field is required for Dynamic Collection." + ); + } else { + const checkFieldsResult = await this.checkFields( + datasetFilePath, + formData.collectionsDynamicField + ); + if (!checkFieldsResult) { + errors.push("Collection's field is not valid"); + } + } + + // Regex check for field + const regex = "^[\\w%\\-]+$"; + if (!String(formData.scopesDynamicField).match(regex)) { + errors.push("scope's field do not match regex"); + } + + if (!String(formData.collectionsDynamicField).match(regex)) { + errors.push("Collection's field do not match regex"); + } + break; + default: + // No additional validation needed for 'defaultCollection' + break; + } + // Return the array of error messages + if (errors.length > 0) { + return errors.join("
"); + } + + return ""; + }; + + async validateKeysAndAdvancedSettingsFormData( + formData: any + ): Promise { + let errors: string[] = []; + + // Validate Keys (No hard check here, we allow any data to pass as it will fail eventually) + if (formData.keyOptions === "fieldValue") { + if (!formData.keyFieldName || formData.keyFieldName.trim() === "") { + errors.push("Field name field is empty."); + } + } else if (formData.keyOptions === "customExpression") { + if ( + !formData.customExpression || + formData.customExpression.trim() === "" + ) { + errors.push("Custom expression field is empty."); + } + } + + // Validate Advanced Settings + if ( + formData.skipDocsOrRows && + String(formData.skipDocsOrRows) !== "" && + parseInt(String(formData.skipDocsOrRows)) < 0 + ) { + // If skipFirstDocuments exists but are less than 0 + errors.push( + "Skip first field does not contain a valid non-negative integer." + ); + } + + if ( + formData.limitDocsOrRows && + String(formData.limitDocsOrRows) !== "" && + parseInt(String(formData.limitDocsOrRows)) < 0 + ) { + // If limitDocsOrRows exists but are less than 0 + errors.push( + "Import up to field does not contain a valid non-negative integer." + ); + } + + if ( + !formData.threads || + !formData.threads.trim() || + parseInt(formData.threads) < 1 + ) { + errors.push("threads cannot be undefined or less than 1"); + } + + // Return the array of error messages + if (errors.length > 0) { + return errors.join("
"); + } + return ""; + } + + public dataImport = async () => { + const connection = getActiveConnection(); + if (!connection) { + return; + } + if (!CBTools.getTool(CBToolsType.CB_IMPORT).isAvailable()) { + vscode.window.showErrorMessage( + "CB Import is still loading, Please try again later" + ); + return; + } + + const dataImportWebviewDetails = + Memory.state.get( + Constants.DATA_IMPORT_WEBVIEW + ); + if (dataImportWebviewDetails) { + // data import webview already exists, Closing existing and creating new + try { + dataImportWebviewDetails.webviewPanel.dispose(); + } catch (e) { + logger.error("Error while disposing data import webview: " + e); + } + Memory.state.update(Constants.DATA_IMPORT_WEBVIEW, null); + } + + const currentPanel = vscode.window.createWebviewPanel( + "dataImport", + "Data Import", + vscode.ViewColumn.One, + { + enableScripts: true, + enableForms: true, + } + ); + Memory.state.update(Constants.DATA_IMPORT_WEBVIEW, { + webviewPanel: currentPanel, + }); + currentPanel.iconPath = { + dark: vscode.Uri.file( + path.join( + __filename, + "..", + "..", + "images", + "dark", + "database-import.svg" + ) + ), + light: vscode.Uri.file( + path.join( + __filename, + "..", + "..", + "images", + "light", + "database-import.svg" + ) + ), + }; + currentPanel.webview.html = getLoader("Data Import"); + + // Get all buckets + const buckets = await connection.cluster?.buckets().getAllBuckets(); + if (buckets === undefined) { + vscode.window.showErrorMessage("Buckets not found"); + return; + } + const bucketNameArr: string[] = []; + for (let bucket of buckets) { + bucketNameArr.push(bucket.name); + } + + try { + currentPanel.webview.html = await getDatasetAndCollection( + bucketNameArr, + undefined, + undefined + ); + currentPanel.webview.onDidReceiveMessage(async (message) => { + switch (message.command) { + // ADD cases here :) + case "vscode-couchbase.tools.dataImport.runImport": { + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; + const datasetAndCollectionData = + message.datasetAndCollectionData; + + CBImport.import({ + bucket: datasetAndCollectionData.bucket, + dataset: datasetAndCollectionData.dataset, + fileFormat: this.fileFormat, + format: this.format, + scopeCollectionExpression: + datasetAndCollectionData.scopeCollectionExpression, + generateKeyExpression: + keysAndAdvancedSettingsData.generateKeyExpression, + skipDocsOrRows: + keysAndAdvancedSettingsData.skipDocsOrRows, + limitDocsOrRows: + keysAndAdvancedSettingsData.limitDocsOrRows, + ignoreFields: + keysAndAdvancedSettingsData.ignoreFields, + threads: keysAndAdvancedSettingsData.threads, + verbose: keysAndAdvancedSettingsData.verboseLog, + }); + + break; + } + case "vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage": { + const keysAndAdvancedSettingsformData = message.data; + const datasetAndCollectionData = + message.datasetAndCollectionData; + const validationError = + await this.validateKeysAndAdvancedSettingsFormData( + keysAndAdvancedSettingsformData + ); + if (validationError === "") { + // Go to summary page + currentPanel.webview.html = + getLoader("Data Import"); + currentPanel.webview.html = await getSummary( + datasetAndCollectionData, + keysAndAdvancedSettingsformData + ); + } else { + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.dataImport.getKeysAndAdvancedSettingsPageFormValidationError", + error: validationError, + }); + } + break; + } + case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage": { + let formData = message.data; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; + const validationError = + await this.validateDatasetAndCollectionFormData( + formData + ); + if (validationError === "") { + // NO Validation Error on Page 1, We can shift to next page + currentPanel.webview.html = + getLoader("Data Import"); + currentPanel.webview.html = + getKeysAndAdvancedSettings( + formData, + keysAndAdvancedSettingsData + ); + } else { + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.dataImport.getDatasetAndCollectionPageFormValidationError", + error: validationError, + }); + } + break; + } + case "vscode-couchbase.tools.dataImport.getScopes": { + const scopes = await getScopes( + message.bucketId, + connection + ); + if (scopes === undefined) { + vscode.window.showErrorMessage( + "Scopes are undefined" + ); + break; + } + + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.dataImport.scopesInfo", + scopes: scopes, + }); + break; + } + case "vscode-couchbase.tools.dataImport.getDatasetFile": { + const options: vscode.OpenDialogOptions = { + canSelectMany: false, + openLabel: "Choose Dataset File", + canSelectFiles: true, + canSelectFolders: false, + }; + + vscode.window + .showOpenDialog(options) + .then((fileUri) => { + if (fileUri && fileUri[0]) { + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.dataImport.datasetFile", + dataset: fileUri[0].fsPath, + }); + } + }); + break; + } + case "vscode-couchbase.tools.dataImport.getKeysBack": { + const datasetAndTargetData = + message.datasetAndTargetData; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; + currentPanel.webview.html = getLoader("Data Import"); + currentPanel.webview.html = + await getDatasetAndCollection( + bucketNameArr, + datasetAndTargetData, + keysAndAdvancedSettingsData + ); + break; + } + case "vscode-couchbase.tools.dataImport.fetchKeyPreview": { + const keyType = message.keyType; + const keyExpr = message.keyExpr; + const preview = await this.updateKeyPreview( + keyType, + keyExpr + ); + currentPanel.webview.postMessage({ + command: + "vscode-couchbase.tools.dataImport.sendKeyPreview", + preview: preview, + }); + break; + } + case "vscode-couchbase.tools.dataImport.onBackSummary": { + const datasetAndCollectionData = + message.datasetAndCollectionData; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; + currentPanel.webview.html = getLoader("Data Import"); + currentPanel.webview.html = getKeysAndAdvancedSettings( + datasetAndCollectionData, + keysAndAdvancedSettingsData + ); + break; + } + } + }); + } catch (err) { + logger.error(`Failed to open data export webview`); + logger.debug(err); + vscode.window.showErrorMessage( + "Failed to open data export webview: " + err + ); + } + + currentPanel.onDidDispose(() => { + Memory.state.update(Constants.DATA_EXPORT_WEBVIEW, null); + }); + }; +} diff --git a/src/extension.ts b/src/extension.ts index 2058e401..5c648f4d 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -65,6 +65,7 @@ import { clearDocumentFilter } from "./commands/documents/clearDocumentFilter"; import { getClusterOverviewData } from "./util/OverviewClusterUtils/getOverviewClusterData"; import { checkAndCreatePrimaryIndex } from "./commands/indexes/checkAndCreatePrimaryIndex"; import { dataExport } from "./pages/Tools/DataExport/dataExport"; +import { DataImport } from "./commands/tools/dataImport"; import { ddlExport } from "./commands/tools/ddlExport/ddlExport"; export function activate(context: vscode.ExtensionContext) { @@ -456,6 +457,15 @@ export function activate(context: vscode.ExtensionContext) { ) ); + subscriptions.push( + vscode.commands.registerCommand( + Commands.dataImport, + async () => { + await new DataImport().dataImport(); + } + ) + ); + subscriptions.push( vscode.commands.registerCommand(Commands.queryContext, () => { fetchQueryContext(workbench, context); diff --git a/src/pages/Tools/DataExport/dataExport.ts b/src/pages/Tools/DataExport/dataExport.ts index 11978bb7..32b4caa0 100644 --- a/src/pages/Tools/DataExport/dataExport.ts +++ b/src/pages/Tools/DataExport/dataExport.ts @@ -13,7 +13,7 @@ import { getLoader } from "../../../webViews/loader.webview"; import { dataExportWebview } from "../../../webViews/tools/dataExport.webview"; import { IConnection } from "../../../types/IConnection"; -const getScopes = async (bucketId: string, connection: IConnection) => { +export const getScopes = async (bucketId: string, connection: IConnection) => { const scopes = await connection.cluster ?.bucket(bucketId) .collections() @@ -55,7 +55,7 @@ const validateFormData = (formData: any): string => { errors.push("Please inform the file destination folder"); } - if (!formData.threads.trim() || parseInt(formData.threads) < 1) { + if(!formData.threads || !formData.threads.trim() || parseInt(formData.threads)<1){ errors.push("threads cannot be undefined or less than 1"); } diff --git a/src/pages/overviewCluster/overviewCluster.ts b/src/pages/overviewCluster/overviewCluster.ts index 5451d3d9..3c17e877 100644 --- a/src/pages/overviewCluster/overviewCluster.ts +++ b/src/pages/overviewCluster/overviewCluster.ts @@ -3,7 +3,6 @@ import * as path from "path"; import { getClusterOverview } from "../../webViews/clusterOverview.webview"; import { ClusterConnectionNode } from "../../model/ClusterConnectionNode"; import { Memory } from "../../util/util"; -import { IConnection } from "../../types/IConnection"; import { logger } from "../../logger/logger"; import { Constants } from "../../util/constants"; import { getClusterOverviewData } from "../../util/OverviewClusterUtils/getOverviewClusterData"; diff --git a/src/tools/CBExport.ts b/src/tools/CBExport.ts index dc612f61..cffe19a0 100644 --- a/src/tools/CBExport.ts +++ b/src/tools/CBExport.ts @@ -62,6 +62,7 @@ export class CBExport { } try { + // Build Command const cmd: string[] = []; cmd.push(CBTools.getTool(Type.CB_EXPORT).path); cmd.push("json"); @@ -97,7 +98,8 @@ export class CBExport { cmd.push("-v"); } - const terminal = vscode.window.createTerminal("CBExport",); + // Run Command + const terminal = vscode.window.createTerminal("CBExport"); let text = cmd.join(" "); terminal.sendText(text); terminal.show(); diff --git a/src/tools/CBImport.ts b/src/tools/CBImport.ts new file mode 100644 index 00000000..4295178f --- /dev/null +++ b/src/tools/CBImport.ts @@ -0,0 +1,142 @@ +import { getActiveConnection, getConnectionId } from "../util/connections"; +import * as vscode from 'vscode'; +import * as keytar from "keytar"; +import { Constants } from "../util/constants"; +import { logger } from "../logger/logger"; +import { IConnection } from "../types/IConnection"; +import { CBTools, Type } from "../util/DependencyDownloaderUtils/CBTool"; + +export interface ICBImportData { + bucket: string; + dataset: string; + fileFormat: string; + scopeCollectionExpression: string; + generateKeyExpression: string; + skipDocsOrRows: string | undefined; + limitDocsOrRows: string | undefined; + ignoreFields: string | undefined; + threads: number; + verbose: boolean; + format: string; +} + +export class CBImport { + + static async import(importData: ICBImportData): Promise { + const connection = getActiveConnection(); + if(!connection){ + return; + } + + let cmd: string[] | Error; + // CMD Builder + try { + cmd = await this.cmdBuilder(importData, connection); + if (cmd instanceof Error) { + throw cmd; + } + } catch(err) { + logger.error("Error while building command for CB Import, please check values and try again"); + logger.debug(err); + vscode.window.showErrorMessage("Error while building command for CB Import, please check values and try again, err: "+err); + return; + } + + // CMD Runner + try { + const terminal = vscode.window.createTerminal("CBImport"); + let text = cmd.join(" "); + logger.info("CB Import Command to run: "+ text); + + terminal.sendText(text); + terminal.show(); + + } catch(err) { + logger.error("Error while running command for CB Import"); + logger.debug(err); + vscode.window.showErrorMessage("Error while running command for CB Import, err: "+err); + } + } + + static async cmdBuilder(importData: ICBImportData, connection: IConnection): Promise { + + const password = await keytar.getPassword(Constants.extensionID, getConnectionId(connection)); + if (!password) { + return new Error("Password not found"); + } + + const cmd: string[] = []; + cmd.push(CBTools.getTool(Type.CB_IMPORT).path); + cmd.push(importData.fileFormat); + cmd.push("--no-ssl-verify"); + + // Cluster details + cmd.push("-c"); + cmd.push(connection.url); + cmd.push("-u"); + cmd.push(connection.username); + cmd.push("-p"); + cmd.push(password); + cmd.push("-b"); + cmd.push(importData.bucket); + + // Dataset details + cmd.push("--dataset"); + cmd.push(`\"file://${importData.dataset}\"`); + + if(importData.fileFormat === "json"){ // If JSON File Format, Lines or Lists needs to be specified as file type + cmd.push("--format"); + cmd.push(importData.format); + } + if(importData.fileFormat === "csv"){ // Field Seperator is taken as ',' by default and only required in case of CSV File Format + cmd.push("--field-separator"); + cmd.push(","); + cmd.push("--infer-types"); // Adding infer types flag as well for csv + } + + // Target Details + cmd.push("--scope-collection-exp"); + cmd.push(importData.scopeCollectionExpression); + + // Key Details + cmd.push("--generate-key"); + cmd.push(importData.generateKeyExpression); + + cmd.push("--generator-delimiter"); // Using default generator delimiter of '#' + cmd.push("#"); + + // Advanced Settings + if(importData.skipDocsOrRows && importData.skipDocsOrRows.trim() !== ""){ + if(importData.fileFormat === "json") { + cmd.push("--skip-docs"); + } else if(importData.fileFormat === "csv") { + cmd.push("--skip-rows"); + } + cmd.push(importData.skipDocsOrRows); + } + + if(importData.limitDocsOrRows && importData.limitDocsOrRows.trim() !== ""){ + if(importData.fileFormat === "json") { + cmd.push("--limit-docs"); + } else if(importData.fileFormat === "csv") { + cmd.push("--limit-rows"); + } + cmd.push(importData.limitDocsOrRows); + } + + if(importData.ignoreFields && importData.ignoreFields.trim() !== ""){ + cmd.push("--ignore-fields"); + cmd.push(importData.ignoreFields); + } + + cmd.push("--threads"); + cmd.push(importData.threads.toString()); + + if (importData.verbose) { + cmd.push("-v"); + } + + return cmd; + + } +} \ No newline at end of file diff --git a/src/util/constants.ts b/src/util/constants.ts index 61bc57f5..d76d3f5a 100644 --- a/src/util/constants.ts +++ b/src/util/constants.ts @@ -97,4 +97,5 @@ export class Constants { public static CURRENT_VBUCKET_REPLICA_ITEMS = "Current vBucket Replica Items"; public static QUERY_RESULT = "Couchbase Query Result"; public static DATA_EXPORT_WEBVIEW = "dataExportWebview"; + public static DATA_IMPORT_WEBVIEW = "dataImportWebview"; } diff --git a/src/webViews/favoriteQueries.webiew.ts b/src/webViews/favoriteQueries.webiew.ts index 974a753a..3a0c85b3 100644 --- a/src/webViews/favoriteQueries.webiew.ts +++ b/src/webViews/favoriteQueries.webiew.ts @@ -132,12 +132,12 @@ export const showFavoriteQueries = (): string => { } .favorite-query-paste-button { - background-color: var(--vscode-button-background); - color: var(--vscode-button-foreground); + background: #ea2328; + color: #eee; } .favorite-query-paste-button:hover { - background-color: var(--vscode-button-hoverBackground); + background: #bb1117; } diff --git a/src/webViews/tools/dataExport.webview.ts b/src/webViews/tools/dataExport.webview.ts index c75cc8e9..f71ebaee 100644 --- a/src/webViews/tools/dataExport.webview.ts +++ b/src/webViews/tools/dataExport.webview.ts @@ -1,5 +1,5 @@ export const dataExportWebview = async (buckets: string[]): Promise => { - return ` + return /*html*/` @@ -7,7 +7,6 @@ export const dataExportWebview = async (buckets: string[]): Promise => { Data Export - + + +

Import Data

+
+
+ Dataset +
+
+ + +
+
Choose
+ +
+
+ +
+ Target Location +
+
+ + + +
+ + + +
+ + + + + + + +
+ + +
+ + + + `; +}; diff --git a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts new file mode 100644 index 00000000..b454ad86 --- /dev/null +++ b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts @@ -0,0 +1,406 @@ +export const getKeysAndAdvancedSettings = ( + datasetAndCollectionData: any, + prefilledData: any +): string => { + return /*html*/ ` + + + + + + Data Import + + + +

Import Data

+
+
+ Document Key +
+
+ + + +
+ + +
+ + +
+ + +
+
Fetch preview
+ +
+
+ +
+
+ ▶️ + Advanced Settings +
+
+ + + +
+ + + +
+ + + +
+ + + +
+
+ + +
+
+
+
+
+ + +
+
+ + + + + + `; +}; diff --git a/src/webViews/tools/dataImport/getSummary.webview.ts b/src/webViews/tools/dataImport/getSummary.webview.ts new file mode 100644 index 00000000..b44f1e29 --- /dev/null +++ b/src/webViews/tools/dataImport/getSummary.webview.ts @@ -0,0 +1,275 @@ +export const getSummary = async ( + datasetAndCollectionData: any, + keysAndAdvancedSettingsData: any +) => { + // Scope And Collection Data Processing for summary + let scopeAndCollectionValue = ""; + let scopeAndCollectionExpr = + datasetAndCollectionData.scopeCollectionExpression; + const scopeAndCollectionType = + datasetAndCollectionData.scopesAndCollections; + let showExtraScopeAndCollection = false; + let scopeData = ""; + let collectionData = ""; + + if (scopeAndCollectionType === "defaultCollection") { + scopeAndCollectionValue = "Default Scope and Collection"; + } else if (scopeAndCollectionType === "SpecifiedCollection") { + scopeAndCollectionValue = "Specifed Scope and Collection"; + showExtraScopeAndCollection = true; + scopeData = datasetAndCollectionData.scopesDropdown; + collectionData = datasetAndCollectionData.collectionsDropdown; + } else if (scopeAndCollectionType === "dynamicCollection") { + scopeAndCollectionValue = "Dynamic Scope and Collection"; + showExtraScopeAndCollection = true; + scopeData = datasetAndCollectionData.scopesDynamicField; + collectionData = datasetAndCollectionData.collectionsDynamicField; + } + + // Document Keys Data Processing for summary + let documentKeyValue = ""; + let keyField = ""; + let keyExpr = keysAndAdvancedSettingsData.generateKeyExpression; + let showExtraKeyField = false; + const documentKeyType = keysAndAdvancedSettingsData.keyOptions; + + if (documentKeyType === "random") { + documentKeyValue = "Generate Random UUID for each Document"; + } else if (documentKeyType === "fieldValue") { + documentKeyValue = "Use the value of field as the key"; + showExtraKeyField = true; + keyField = keysAndAdvancedSettingsData.keyFieldName; + } else if (documentKeyType === "customExpression") { + documentKeyValue = "Generate key based on custom expression"; + } + + // Advanced Settings + + return /*html*/ ` + + + + + + Data Import + + +

Import Data

+
+

Summary

+
+
Dataset Path:
+
${ + datasetAndCollectionData.dataset + }
+
+ +
+
Bucket Name:
+
${ + datasetAndCollectionData.bucket + }
+
+ +
+
Scopes and Collections:
+
${scopeAndCollectionValue}
+ ${ + showExtraScopeAndCollection === true + ? `
+
- Scope:
+
${scopeData}
+ +
+
- Collection:
+
${collectionData}
+ ` + : `` + } +
+
- Scope and Collection Expression:
+
${scopeAndCollectionExpr}
+
+ +
+
Document Key:
+
${documentKeyValue}
+ ${ + showExtraKeyField === true + ? `
+
- Key Field:
+
${keyField}
+ ` + : `` + } +
+
- Key Expression:
+
${keyExpr}
+
+ +
+
Advanced Settings:
+ ${ + keysAndAdvancedSettingsData.skipDocsOrRows && + keysAndAdvancedSettingsData.skipDocsOrRows.trim() !== + "" + ? ` +
+
- Skip First Documents:
+
${keysAndAdvancedSettingsData.skipDocsOrRows}
+ ` + : `` + } + ${ + keysAndAdvancedSettingsData.limitDocsOrRows && + keysAndAdvancedSettingsData.limitDocsOrRows.trim() !== + "" + ? ` +
+
- Limit Documents/Rows:
+
${keysAndAdvancedSettingsData.limitDocsOrRows}
+ ` + : `` + } + ${ + keysAndAdvancedSettingsData.ignoreFields && + keysAndAdvancedSettingsData.ignoreFields.trim() !== + "" + ? ` +
+
- Ignore Fields:
+
${keysAndAdvancedSettingsData.ignoreFields}
+ ` + : `` + } + ${ + keysAndAdvancedSettingsData.threads && + keysAndAdvancedSettingsData.threads.trim() !== "" + ? ` +
+
- Threads:
+
${keysAndAdvancedSettingsData.threads}
+ ` + : `` + } + +
+
- Verbose Log:
+
${ + keysAndAdvancedSettingsData.verboseLog + }
+
+ +
+ + +
+
+ + + + + + `; +};