From 419aca81a9948441d016c0be173b495f12049640 Mon Sep 17 00:00:00 2001 From: Lokesh Goel Date: Wed, 25 Oct 2023 19:16:34 +0530 Subject: [PATCH] add fetch key preview functionality --- package-lock.json | 99 +++- package.json | 12 +- src/commands/tools/dataImport.ts | 520 ++++++++++++++++-- src/tools/CBImport.ts | 8 +- .../getKeysAndAdvancedSettings.webview.ts | 29 +- 5 files changed, 604 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index 821fb305..8334d587 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,13 @@ "@monaco-editor/react": "^4.5.2", "@popperjs/core": "^2.11.8", "@types/decompress": "^4.2.4", + "@types/stream-json": "^1.7.5", "ag-grid-community": "^29.3.5", "ag-grid-react": "^29.3.5", "axios": "^1.4.0", "clsx": "^2.0.0", "couchbase": "^4.2.1", + "csv-parse": "^5.5.2", "d3-hierarchy": "^1.1.9", "d3-selection": "^1.4.2", "d3-shape": "^1.3.7", @@ -24,6 +26,7 @@ "dayjs": "^1.11.9", "decompress": "^4.2.1", "gitly": "^2.4.2", + "jsonstream2": "^3.0.0", "keytar": "^7.7.0", "monaco-editor": "^0.43.0", "node-addon-api": "^6.0.0", @@ -33,7 +36,9 @@ "react-merge-refs": "^2.0.2", "react-popper": "^2.3.0", "react-popper-tooltip": "^4.4.2", - "react-router-dom": "^6.14.2" + "react-router-dom": "^6.14.2", + "stream-json": "^1.8.0", + "uuid": "^9.0.1" }, "devDependencies": { "@babel/parser": "^7.22.7", @@ -44,6 +49,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "@types/tar": "^6.1.5", + "@types/uuid": "^9.0.6", "@types/vscode": "^1.63.1", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", @@ -743,6 +749,23 @@ "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", "dev": true }, + "node_modules/@types/stream-chain": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.3.tgz", + "integrity": "sha512-cwWE6mrdDpmW3B5wr1+vpjbg8h3hZfOr/PbKQ38VE21xNyF64GNjrh855YdNAPCt4kSYXuLwgRqBWkY/dD6KMg==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stream-json": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.5.tgz", + "integrity": "sha512-IVTtojYNqc6RT9FWBlwPLG6QTVdv2gHdqHOyBYPgcsCKfwIxcQpKw0e0ybQVyCvJJT9Le6w9RB5r5c6mo2m+IQ==", + "dependencies": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "node_modules/@types/tar": { "version": "6.1.5", "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.5.tgz", @@ -762,6 +785,12 @@ "node": ">=8" } }, + "node_modules/@types/uuid": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.6.tgz", + "integrity": "sha512-BT2Krtx4xaO6iwzwMFUYvWBWkV2pr37zD68Vmp1CDV196MzczBRxuEpD6Pr395HAgebC/co7hOphs53r8V7jew==", + "dev": true + }, "node_modules/@types/vscode": { "version": "1.74.0", "dev": true, @@ -2510,6 +2539,11 @@ "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", "dev": true }, + "node_modules/csv-parse": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.2.tgz", + "integrity": "sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==" + }, "node_modules/d3-color": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.4.1.tgz", @@ -4931,6 +4965,30 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "engines": [ + "node >= 0.2.0" + ] + }, + "node_modules/jsonstream2": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/jsonstream2/-/jsonstream2-3.0.0.tgz", + "integrity": "sha512-8ngq2XB8NjYrpe3+Xtl9lFJl6RoV2dNT4I7iyaHwxUpTBwsj0AlAR7epGfeYVP0z4Z7KxMoSxRgJWrd2jmBT/Q==", + "dependencies": { + "jsonparse": "1.3.1", + "through2": "^3.0.1", + "type-component": "0.0.1" + }, + "bin": { + "jsonstream": "bin/jsonstream.js" + }, + "engines": { + "node": ">=5.10.0" + } + }, "node_modules/keytar": { "version": "7.9.0", "hasInstallScript": true, @@ -8143,6 +8201,19 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==" + }, + "node_modules/stream-json": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.8.0.tgz", + "integrity": "sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", @@ -8486,6 +8557,15 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, + "node_modules/through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + } + }, "node_modules/tmp": { "version": "0.2.1", "dev": true, @@ -8619,6 +8699,11 @@ "node": ">= 0.8.0" } }, + "node_modules/type-component": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/type-component/-/type-component-0.0.1.tgz", + "integrity": "sha512-mDZRBQS2yZkwRQKfjJvQ8UIYJeBNNWCq+HBNstl9N5s9jZ4dkVYXEGkVPsSCEh5Ld4JM1kmrZTzjnrqSAIQ7dw==" + }, "node_modules/type-fest": { "version": "0.20.2", "dev": true, @@ -8835,6 +8920,18 @@ "version": "1.0.2", "license": "MIT" }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "dev": true, diff --git a/package.json b/package.json index 1b74671c..b57df6d6 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "@types/tar": "^6.1.5", + "@types/uuid": "^9.0.6", "@types/vscode": "^1.63.1", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", @@ -82,11 +83,13 @@ "@monaco-editor/react": "^4.5.2", "@popperjs/core": "^2.11.8", "@types/decompress": "^4.2.4", + "@types/stream-json": "^1.7.5", "ag-grid-community": "^29.3.5", "ag-grid-react": "^29.3.5", "axios": "^1.4.0", "clsx": "^2.0.0", "couchbase": "^4.2.1", + "csv-parse": "^5.5.2", "d3-hierarchy": "^1.1.9", "d3-selection": "^1.4.2", "d3-shape": "^1.3.7", @@ -94,6 +97,7 @@ "dayjs": "^1.11.9", "decompress": "^4.2.1", "gitly": "^2.4.2", + "jsonstream2": "^3.0.0", "keytar": "^7.7.0", "monaco-editor": "^0.43.0", "node-addon-api": "^6.0.0", @@ -103,7 +107,9 @@ "react-merge-refs": "^2.0.2", "react-popper": "^2.3.0", "react-popper-tooltip": "^4.4.2", - "react-router-dom": "^6.14.2" + "react-router-dom": "^6.14.2", + "stream-json": "^1.8.0", + "uuid": "^9.0.1" }, "activationEvents": [ "onFileSystem:couchbase", @@ -583,7 +589,7 @@ }, { "title": "Refresh Cluster Overview", - "command":"vscode-couchbase.refreshClusterOverview", + "command": "vscode-couchbase.refreshClusterOverview", "icon": "$(refresh)", "category": "Couchbase" }, @@ -949,4 +955,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/commands/tools/dataImport.ts b/src/commands/tools/dataImport.ts index fe16db14..78988dad 100644 --- a/src/commands/tools/dataImport.ts +++ b/src/commands/tools/dataImport.ts @@ -14,22 +14,32 @@ import * as fs from "fs"; import { getScopes } from "../../pages/Tools/DataExport/dataExport"; import { getKeysAndAdvancedSettings } from "../../webViews/tools/dataImport/getKeysAndAdvancedSettings.webview"; import { CBImport } from "../../tools/CBImport"; +import { parser } from "stream-json"; +import { pick } from "stream-json/filters/Pick"; +import { streamValues } from "stream-json/streamers/StreamValues"; +import * as csv from "csv-parse"; +import { v4 as uuidv4 } from 'uuid'; interface IDataImportWebviewState { webviewPanel: vscode.WebviewPanel; } - export class DataImport { cachedJsonDocs: string[] = []; cachedCsvDocs: Map = new Map(); - PREVIEW_SIZE: number = 10; // Define your desired preview size + PREVIEW_SIZE: number = 6; // Define your desired preview size JSON_FILE_EXTENSION: string = ".json"; CSV_FILE_EXTENSION: string = ".csv"; - constructor( - public datasetField: string, - PREVIEW_SIZE: number = 6 - ) {} - + JSON_FILE_FORMAT: string = "json"; + CSV_FILE_FORMAT: string = "csv"; + datasetField: string = ""; + readonly UUID_FLAG = "#UUID#"; + readonly MONO_INCR_FLAG = "#MONO_INCR#"; + readonly WORDS_WITH_PERCENT_SYMBOLS_REGEX = "%(\\w+)%"; + + protected fileFormat: string = ""; + constructor() {} + + // Functions to fetch key previews protected cleanString(input: string): string { const firstOpenBracket = input.indexOf("{"); const lastCloseBracket = input.lastIndexOf("}"); @@ -63,10 +73,10 @@ export class DataImport { return stack.length === 0; } - private readAndProcessPartialDataFromDataset = () => { + private readAndProcessPartialDataFromDataset = async () => { try { const datasetPath: string = this.datasetField; - + console.log("datasetPath",datasetPath); if (datasetPath.endsWith(this.JSON_FILE_EXTENSION)) { const readStream = fs.createReadStream(datasetPath, { encoding: "utf8", @@ -77,6 +87,7 @@ export class DataImport { readStream.on("data", (chunk: string) => { const lines = chunk.split("\n"); + for (let line of lines) { if (counter === 0 && !insideArray) { @@ -105,63 +116,440 @@ export class DataImport { if (this.isValidBrackets(documentFound)) { documentFound = this.cleanString(documentFound); this.cachedJsonDocs.push(documentFound); + documentFound = ""; } } }); readStream.on("end", () => { - readStream.close(); + readStream.close(); }); } } catch (err) {} }; - validateFormData = (formData: any): string => { + + async updateKeyPreview(keyType: string, keyExpr: string): Promise { + if ((this.fileFormat === this.JSON_FILE_FORMAT && this.cachedJsonDocs.length === 0) || + (this.fileFormat === this.CSV_FILE_FORMAT && this.cachedCsvDocs.size === 0)) { + await this.readAndProcessPartialDataFromDataset(); + } + + const previewContent: string[] = []; + let monoIncrValue = 1; + + if (keyType === "fieldValue") { + const fieldName = keyExpr; + if (this.fileFormat === this.JSON_FILE_FORMAT) { + for (let i = 0; i < Math.min(this.cachedJsonDocs.length, this.PREVIEW_SIZE); i++) { + const jsonObject = JSON.parse(this.cachedJsonDocs[i]); + if (jsonObject.hasOwnProperty(fieldName)) { + previewContent.push(jsonObject[fieldName].toString()); + } + } + } else if (this.fileFormat === this.CSV_FILE_FORMAT && this.cachedCsvDocs.get(fieldName) !== null) { + for (let i = 0; i < Math.min(this.cachedCsvDocs.size, this.PREVIEW_SIZE); i++) { + let currentContext = this.cachedCsvDocs.get(fieldName); + if(currentContext){ + previewContent.push(currentContext[i].toString()); + } + } + } + } else if (keyType === "customExpression") { + if (this.fileFormat === this.JSON_FILE_FORMAT) { + const expression = keyExpr; + const pattern = new RegExp(this.WORDS_WITH_PERCENT_SYMBOLS_REGEX, 'g'); + let matches: string[] = []; + + let match; + console.log("before while"); + while ((match = pattern.exec(expression)) !== null) { + // match[1] contains the captured word (the part between % symbols) + console.log(match); + if(match[1] === ""){ + break; + } + matches.push(match[1]); + } + const fieldNamesList: string[] = []; + matches.forEach(match => { + fieldNamesList.push(match.replace(/%/g, '')); + }); + console.log("matches", matches); + console.log("fieldNamesList", fieldNamesList); + + + for (let i = 0; i < Math.min(this.cachedJsonDocs.length, this.PREVIEW_SIZE); i++) { + const jsonObject = JSON.parse(this.cachedJsonDocs[i]); + let keyBuilder = expression; + + fieldNamesList.forEach(fieldName => { + if (jsonObject.hasOwnProperty(fieldName)) { + keyBuilder = keyBuilder.replace(new RegExp(`%${fieldName}%`,'g'), jsonObject[fieldName].toString()); + } + }); + + keyBuilder = keyBuilder.replace(new RegExp(this.UUID_FLAG, 'g'), uuidv4()); + keyBuilder = keyBuilder.replace(new RegExp(this.MONO_INCR_FLAG, 'g'), monoIncrValue.toString()); + monoIncrValue++; + + previewContent.push(keyBuilder); + } + } else if (this.fileFormat === this.CSV_FILE_FORMAT) { + const expression = keyExpr; + const pattern = new RegExp(this.WORDS_WITH_PERCENT_SYMBOLS_REGEX); + const matcher = expression.match(pattern); + const fieldNamesList: string[] = []; + + if (matcher) { + matcher.forEach(match => { + fieldNamesList.push(match.replace(/%/g, '')); + }); + } + + for (let i = 0; i < Math.min(this.cachedCsvDocs.size, this.PREVIEW_SIZE); i++) { + let keyBuilder = expression; + + fieldNamesList.forEach(fieldName => { + const keyContent = this.cachedCsvDocs.get(fieldName); + if(keyContent){ + keyBuilder = keyBuilder.replace(new RegExp(`%${fieldName}%`,'g'), keyContent[i].toString()); + } + + }); + + keyBuilder = keyBuilder.replace(new RegExp(this.UUID_FLAG, 'g'), uuidv4()); + keyBuilder = keyBuilder.replace(new RegExp(this.MONO_INCR_FLAG, 'g'), monoIncrValue.toString()); + monoIncrValue++; + + previewContent.push(keyBuilder); + } + } + } + console.log(previewContent); + return previewContent.join("\n"); +} + + + + // Functions to detect validity of scopes and collections + + async sampleElementFromJsonArrayFile( + filePath: string + ): Promise { + return new Promise((resolve, reject) => { + const pipeline = fs + .createReadStream(filePath) + .pipe(parser()) + .pipe(pick({ filter: "0" })) + .pipe(streamValues()); + + let result: string | null = null; + + pipeline.on("data", (data) => { + if (result === null) { + result = JSON.stringify(data.value); + pipeline.destroy(); + } + }); + + pipeline.on("close", () => { + resolve(result); + }); + pipeline.on("error", (err) => { + console.error(err); + resolve(null); + }); + }); + } + + async sampleElementFromCsvFile( + filePath: string, + lineNumber: number + ): Promise { + return new Promise((resolve, reject) => { + let currentLine = 0; + let result: string[] | null = null; + fs.createReadStream(filePath) + .pipe(csv.parse()) + .on("data", (row) => { + currentLine++; + if (currentLine === lineNumber) { + result = Object.values(row); + resolve(result); + } + }) + .on("end", () => { + if (result === null) { + resolve([]); + } + }) + .on("error", (err) => { + console.error(err); + resolve([]); + }); + }); + } + + checkFields = async ( + filePath: string, + fieldText: string + ): Promise => { + const pattern = /%(.*?)%/g; + let match; + + while ((match = pattern.exec(fieldText)) !== null) { + const fieldName = match[1]; + try { + if (this.fileFormat === "json") { + const sampleElement = await this.sampleElementFromJsonArrayFile( + filePath + ); + + if (sampleElement !== null) { + const jsonObject = JSON.parse(sampleElement); + if (!jsonObject.hasOwnProperty(fieldName)) { + return false; + } + } else { + return false; + } + } else if (this.fileFormat === "csv") { + const headers = await this.sampleElementFromCsvFile(filePath, 1); + if (headers !== null && !headers.includes(fieldName)) { + return false; + } + } + } catch (e) { + console.error(e); + return false; + } + } + + return true; + }; + + // Functions to detect Validity of dataset + detectDatasetFormat = async (filePath: string): Promise => { + try { + let currentPosition = 0; + const startBuffer = await this.readFirstTwoNonEmptyCharacters(filePath); + const endBuffer = await this.readLastTwoNonEmptyCharacters(filePath); + + const firstChar = startBuffer[1]; + const secondChar = startBuffer[0]; + + const secondLastChar = endBuffer[1]; + const lastChar = endBuffer[0]; + + if ( + firstChar === "[" && + secondChar === "{" && + secondLastChar === "}" && + lastChar === "]" + ) { + return "list"; + } else if (firstChar === "{" && lastChar === "}") { + return "lines"; + } + return null; + } catch (err) { + throw new Error("" + err); + } + }; + + readLastTwoNonEmptyCharacters = async (filePath: string): Promise => { + const chunkSize = 1024; // Adjust the chunk size as needed + let buffer = Buffer.alloc(2); // Initialize a buffer to store the last two non-empty characters + let bytesRead = 0; + let lastTwoNonEmptyChars = ""; + + const fd = await fs.promises.open(filePath, "r"); + const fileSize = (await fd.stat()).size; + + for ( + let position = fileSize - 1; + position >= 0 && bytesRead < 2; + position -= chunkSize + ) { + const bufferSize = Math.min(chunkSize, position + 1); + const chunk = Buffer.alloc(bufferSize); + + await fd.read(chunk, 0, bufferSize, position); + + for (let i = bufferSize - 1; i >= 0; i--) { + const char = chunk.toString("utf8", i, i + 1); + if (!/[\s\t\n]/.test(char) && char.charCodeAt(0) !== 0) { + // If the character is not a space, tab, or newline, add it to the buffer + buffer[1] = buffer[0]; // Shift the previous character + buffer[0] = chunk[i]; // Store the current character + + bytesRead++; + lastTwoNonEmptyChars = buffer.toString("utf8", 0, bytesRead); + } + + if (bytesRead >= 2) { + break; + } + } + } + + await fd.close(); + + return lastTwoNonEmptyChars; + }; + + readFirstTwoNonEmptyCharacters = async ( + filePath: string + ): Promise => { + const chunkSize = 1024; // Adjust the chunk size as needed + let buffer = Buffer.alloc(2); // Initialize a buffer to store the first two non-empty characters + let bytesRead = 0; + + const readStream = fs.createReadStream(filePath, { + highWaterMark: chunkSize, + }); + let closed = false; + for await (const chunk of readStream) { + for (let i = 0; i < chunk.length; i++) { + const char = chunk.toString("utf8", i, i + 1); + + if (!/[\s\t\n]/.test(char)) { + // If the character is not a space, tab, or newline, add it to the buffer + buffer[bytesRead] = chunk[i]; // Store the current character + bytesRead++; + + if (bytesRead >= 2) { + // We have found the first two non-empty characters, so close the stream + readStream.close(); + closed = true; + break; + } + } + } + if (closed) { + break; + } + } + return buffer.toString("utf8", 0, bytesRead); + }; + + validateDataset = async (datasetFilePath: string): Promise => { let errors = []; + // Check if dataset is not empty + if (!datasetFilePath || datasetFilePath.trim() === "") { + errors.push("Dataset is required."); + return errors; + } + logger.debug("Dataset file path received: " + datasetFilePath); + + if (datasetFilePath.endsWith(this.JSON_FILE_EXTENSION)) { + this.fileFormat = this.JSON_FILE_FORMAT; + } else if (datasetFilePath.endsWith(this.CSV_FILE_EXTENSION)) { + this.fileFormat = this.CSV_FILE_FORMAT; + } else { + errors.push("Please enter valid json or csv file only"); + } - // Check if dataset and bucket are not empty - if (!formData.dataset) { - errors.push('Dataset is required.'); + if (this.fileFormat === this.JSON_FILE_FORMAT) { + let format: string | null = await this.detectDatasetFormat( + datasetFilePath + ); + console.log(format); + if (format) { + console.log(`Detected format: ${format}`); + } else { + console.log("Format not detected."); + errors.push("Please enter valid json file format only"); + } + + // Check if given JSON File is correct } + return errors; + }; + + validateFormData = async (formData: any): Promise => { + let errors: string[] = []; + + // Validate Dataset + const datasetFilePath: string = formData.dataset; + const datasetErrors = await this.validateDataset(datasetFilePath); + for (let error of datasetErrors) { + errors.push(error); + } + if(datasetErrors.length === 0){ + this.datasetField = datasetFilePath; + } + + + + // Check if bucket is not empty if (!formData.bucket) { - errors.push('Bucket is required.'); + errors.push("Bucket is required."); } // Perform different validation checks based on the value of scopesAndCollections switch (formData.scopesAndCollections) { - case 'SpecifiedCollection': - // Check if scopesDropdown and collectionsDropdown are not empty - if (!formData.scopesDropdown || formData.scopesDropdown === "") { - errors.push('Scope is required for Specified Collection.'); - } - if (!formData.collectionsDropdown || formData.collectionsDropdown === "") { - errors.push('Collection is required for Specified Collection.'); - } - break; - case 'dynamicCollection': - // Check if scopesDynamicField and collectionsDynamicField are not empty - if (!formData.scopesDynamicField || formData.scopesDynamicField.trim() === "") { - errors.push('Scope Field is required for Dynamic Collection.'); - } - if (!formData.collectionsDynamicField || formData.collectionsDynamicField.trim() === "") { - errors.push('Collection Field is required for Dynamic Collection.'); - } - // TODO: Add advanced check for dynamic fields - break; - default: - // No additional validation needed for 'defaultCollection' - break; + case "SpecifiedCollection": + // Check if scopesDropdown and collectionsDropdown are not empty + if (!formData.scopesDropdown || formData.scopesDropdown === "") { + errors.push("Scope is required for Specified Collection."); + } + if ( + !formData.collectionsDropdown || + formData.collectionsDropdown === "" + ) { + errors.push("Collection is required for Specified Collection."); + } + break; + case "dynamicCollection": + // Check if scopesDynamicField and collectionsDynamicField are not empty + if ( + !formData.scopesDynamicField || + formData.scopesDynamicField.trim() === "" + ) { + errors.push("Scope Field is required for Dynamic Collection."); + } else { + const checkFieldsResult = await this.checkFields(datasetFilePath, formData.scopesDynamicField); + if(!checkFieldsResult){ + errors.push("Scope's field is not valid"); + } + } + if ( + !formData.collectionsDynamicField || + formData.collectionsDynamicField.trim() === "" + ) { + errors.push("Collection Field is required for Dynamic Collection."); + } else { + const checkFieldsResult = await this.checkFields(datasetFilePath, formData.collectionsDynamicField); + if(!checkFieldsResult){ + errors.push("Collection's field is not valid"); + } + } + + // Regex check for field + const regex = "^[\\w%\\-]+$"; + if(!String(formData.scopesDynamicField).match(regex)){ + errors.push("scope's field do not match regex"); + } + + if(!String(formData.collectionsDynamicField).match(regex)){ + errors.push("Collection's field do not match regex"); + } + break; + default: + // No additional validation needed for 'defaultCollection' + break; } // Return the array of error messages if (errors.length > 0) { return errors.join("
"); } - + return ""; }; - public dataImport = async () => { const connection = getActiveConnection(); if (!connection) { @@ -221,40 +609,48 @@ export class DataImport { } try { - currentPanel.webview.html = await getDatasetAndCollection(bucketNameArr, undefined); + currentPanel.webview.html = await getDatasetAndCollection( + bucketNameArr, + undefined + ); currentPanel.webview.onDidReceiveMessage(async (message) => { switch (message.command) { // ADD cases here :) - case 'vscode-couchbase.tools.dataImport.runImport': + case "vscode-couchbase.tools.dataImport.runImport": const runFormData = message.data; - const runValidationError = this.validateFormData(runFormData); - if (runValidationError === "") { + const datasetAndCollectionData = message.datasetAndCollectionData; + const runValidationError = await this.validateFormData(runFormData); + console.log("all data at end"); + console.log(runFormData); + console.log(datasetAndCollectionData); + if (runValidationError === "" || true) { CBImport.import({ - bucket: runFormData.bucket, // TODO: bucket should be taken from other form - dataset: runFormData.dataset, - fileFormat: runFormData.fileFormat, - scopeCollectionExpression: runFormData.scopeCollectionExpression, + bucket: datasetAndCollectionData.bucket, // TODO: bucket should be taken from other form + dataset: datasetAndCollectionData.dataset, + fileFormat: "json", + scopeCollectionExpression: + datasetAndCollectionData.scopeCollectionExpression, generateKeyExpression: runFormData.generateKeyExpression, skipDocsOrRows: runFormData.skipDocsOrRows, limitDocsOrRows: runFormData.limitDocsOrRows, ignoreFields: runFormData.ignoreFields, threads: runFormData.threads, - verbose: runFormData.verbose + verbose: runFormData.verbose, }); } break; - case 'vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage': + case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage": const formData = message.data; - const validationError = this.validateFormData(formData); + const validationError = await this.validateFormData(formData); if (validationError === "") { // NO Validation Error on Page 1, We can shift to next page currentPanel.webview.html = getLoader("Data Import"); currentPanel.webview.html = getKeysAndAdvancedSettings(formData); - } else { currentPanel.webview.postMessage({ - command: "vscode-couchbase.tools.dataImport.getDatasetAndCollectionPageFormValidationError", + command: + "vscode-couchbase.tools.dataImport.getDatasetAndCollectionPageFormValidationError", error: validationError, }); } @@ -288,12 +684,26 @@ export class DataImport { } }); break; - case 'vscode-couchbase.tools.dataImport.getKeysBack': + case "vscode-couchbase.tools.dataImport.getKeysBack": const datasetAndTargetData = message.datasetAndTargetData; - const keysAndAdvancedSettingsData = message.keysAndAdvancedSettingsData; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; currentPanel.webview.html = getLoader("Data Import"); - currentPanel.webview.html = await getDatasetAndCollection(bucketNameArr, datasetAndTargetData); + currentPanel.webview.html = await getDatasetAndCollection( + bucketNameArr, + datasetAndTargetData + ); break; + case "vscode-couchbase.tools.dataImport.fetchKeyPreview": + console.log("inside data import"); + const keyType = message.keyType; + const keyExpr = message.keyExpr; + const preview = await this.updateKeyPreview(keyType, keyExpr); + currentPanel.webview.postMessage({ + command: "vscode-couchbase.tools.dataImport.sendKeyPreview", + preview: preview + + }); } }); } catch (err) { diff --git a/src/tools/CBImport.ts b/src/tools/CBImport.ts index 38ee9a46..b3056185 100644 --- a/src/tools/CBImport.ts +++ b/src/tools/CBImport.ts @@ -26,6 +26,7 @@ export class CBImport { if(!connection){ return; } + console.log(importData); let cmd: string[] | Error; // CMD Builder try { @@ -33,6 +34,7 @@ export class CBImport { if (cmd instanceof Error) { throw cmd; } + console.log(cmd); } catch(err) { logger.error("Error while building command for CB Import, please check values and try again"); logger.debug(err); @@ -46,8 +48,8 @@ export class CBImport { let text = cmd.join(" "); console.log(text); - // terminal.sendText(text); - // terminal.show(); + terminal.sendText(text); + terminal.show(); } catch(err) { logger.error("Error while running command for CB Import"); @@ -80,7 +82,7 @@ export class CBImport { // Dataset details cmd.push("--dataset"); - cmd.push(`file://${importData.dataset}`); + cmd.push(`\"file://${importData.dataset}\"`); if(importData.fileFormat === "JSON"){ // If JSON File Format, Lines or Arrays needs to be specified as file type cmd.push("--format"); diff --git a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts index bf14d606..5d2daf00 100644 --- a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts +++ b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts @@ -197,7 +197,7 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => {
-
Fetch preview
+
Fetch preview

@@ -261,6 +261,20 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { }); }); + function fetchKeyPreview() { + + var keyOptions = document.getElementById('keyOptions').value; + var keyFieldName = document.getElementById('keyFieldName').value; + var customExpression = document.getElementById('customExpression').value; + + + vscode.postMessage({ + command: "vscode-couchbase.tools.dataImport.fetchKeyPreview", + keyType: keyOptions, + keyExpr: keyOptions === "fieldValue" ? keyFieldName : (keyOptions === "customExpression" ? customExpression : undefined) + }) + } + function onNextClick(event) { event.preventDefault(); // prevent form submission @@ -288,7 +302,8 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { vscode.postMessage({ command: 'vscode-couchbase.tools.dataImport.runImport', - data: formData + data: formData, + datasetAndCollectionData: ${JSON.stringify(lastPageData)} }); } @@ -322,6 +337,16 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { datasetAndTargetData: lastPageData }) } + + window.addEventListener('message', event => { + const message = event.data; // The JSON data our extension sent + switch (message.command) { + case "vscode-couchbase.tools.dataImport.sendKeyPreview": + let preview = message.preview; + document.getElementById("keyPreviewTextArea").innerHTML = preview; + break; + } + })