From a0df0638e30509bfa6acf0714d2c1fec5ebf8ddf Mon Sep 17 00:00:00 2001 From: Lokesh Goel Date: Fri, 27 Oct 2023 18:53:12 +0530 Subject: [PATCH] fixed csv import issues --- src/commands/tools/dataImport.ts | 114 +++++++++++++----- src/tools/CBImport.ts | 2 +- .../getDatasetAndCollection.webview.ts | 28 +++-- .../getKeysAndAdvancedSettings.webview.ts | 52 +++++++- .../tools/dataImport/getSummary.webview.ts | 100 +++++++++------ 5 files changed, 215 insertions(+), 81 deletions(-) diff --git a/src/commands/tools/dataImport.ts b/src/commands/tools/dataImport.ts index aa764c44..64157ec4 100644 --- a/src/commands/tools/dataImport.ts +++ b/src/commands/tools/dataImport.ts @@ -27,7 +27,7 @@ interface IDataImportWebviewState { export class DataImport { cachedJsonDocs: string[] = []; cachedCsvDocs: Map = new Map(); - PREVIEW_SIZE: number = 6; // Define your desired preview size + PREVIEW_SIZE: number = 6; JSON_FILE_EXTENSION: string = ".json"; CSV_FILE_EXTENSION: string = ".csv"; JSON_FILE_FORMAT: string = "json"; @@ -96,7 +96,6 @@ export class DataImport { if (counter === 0 && !insideArray) { if (!line.trim().startsWith("[")) { logger.debug("Not a JSON array"); - // TODO: Give error to user as well readStream.close(); return; } @@ -128,8 +127,49 @@ export class DataImport { readStream.on("end", () => { readStream.close(); }); + } else { + const headers = await this.sampleElementFromCsvFile( + datasetPath, + 1 + ); + if (headers === null) { + return; + } + for ( + let lineNumber = 2; + lineNumber < 2 + this.PREVIEW_SIZE; + lineNumber++ + ) { + const data = await this.sampleElementFromCsvFile( + datasetPath, + lineNumber + ); + if (data === null) { + continue; + } + + for ( + let headersIndex = 0; + headersIndex < headers.length; + headersIndex++ + ) { + let cachedCsv = this.cachedCsvDocs.get( + headers[headersIndex] + ); + if (!cachedCsv) { + cachedCsv = new Array(this.PREVIEW_SIZE); + } + cachedCsv[lineNumber - 2] = data[headersIndex]; + this.cachedCsvDocs.set( + headers[headersIndex], + cachedCsv + ); + } + } } - } catch (err) {} + } catch (err) { + logger.error(err); + } }; async updateKeyPreview(keyType: string, keyExpr: string): Promise { @@ -140,6 +180,10 @@ export class DataImport { this.cachedCsvDocs.size === 0) ) { await this.readAndProcessPartialDataFromDataset(); + // Wait 500ms extra to cache everything + await new Promise((resolve) => { + setTimeout(resolve, 500); + }); } const previewContent: string[] = []; @@ -675,14 +719,6 @@ export class DataImport { ); } - if ( - formData.ignoreFields && - String(formData.ignoreFields).trim() !== "" - ) { - // Ignore fields exists, validating it - // TODO: Add validations for CSV File format - } - if ( !formData.threads || !formData.threads.trim() || @@ -774,35 +810,40 @@ export class DataImport { try { currentPanel.webview.html = await getDatasetAndCollection( bucketNameArr, + undefined, undefined ); currentPanel.webview.onDidReceiveMessage(async (message) => { switch (message.command) { // ADD cases here :) case "vscode-couchbase.tools.dataImport.runImport": { - const keysAndAdvancedSettingsData = message.keysAndAdvancedSettingsData; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; const datasetAndCollectionData = message.datasetAndCollectionData; CBImport.import({ - bucket: datasetAndCollectionData.bucket, // TODO: bucket should be taken from other form + bucket: datasetAndCollectionData.bucket, dataset: datasetAndCollectionData.dataset, fileFormat: this.fileFormat, format: this.format, scopeCollectionExpression: datasetAndCollectionData.scopeCollectionExpression, generateKeyExpression: - keysAndAdvancedSettingsData.generateKeyExpression, - skipDocsOrRows: keysAndAdvancedSettingsData.skipDocsOrRows, - limitDocsOrRows: keysAndAdvancedSettingsData.limitDocsOrRows, - ignoreFields: keysAndAdvancedSettingsData.ignoreFields, + keysAndAdvancedSettingsData.generateKeyExpression, + skipDocsOrRows: + keysAndAdvancedSettingsData.skipDocsOrRows, + limitDocsOrRows: + keysAndAdvancedSettingsData.limitDocsOrRows, + ignoreFields: + keysAndAdvancedSettingsData.ignoreFields, threads: keysAndAdvancedSettingsData.threads, verbose: keysAndAdvancedSettingsData.verboseLog, }); break; } - case "vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage":{ + case "vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage": { const keysAndAdvancedSettingsformData = message.data; const datasetAndCollectionData = message.datasetAndCollectionData; @@ -814,8 +855,10 @@ export class DataImport { // Go to summary page currentPanel.webview.html = getLoader("Data Import"); - currentPanel.webview.html = await getSummary(datasetAndCollectionData, keysAndAdvancedSettingsformData); - + currentPanel.webview.html = await getSummary( + datasetAndCollectionData, + keysAndAdvancedSettingsformData + ); } else { currentPanel.webview.postMessage({ command: @@ -825,8 +868,10 @@ export class DataImport { } break; } - case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage":{ + case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage": { let formData = message.data; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; const validationError = await this.validateDatasetAndCollectionFormData( formData @@ -836,7 +881,10 @@ export class DataImport { currentPanel.webview.html = getLoader("Data Import"); currentPanel.webview.html = - getKeysAndAdvancedSettings(formData); + getKeysAndAdvancedSettings( + formData, + keysAndAdvancedSettingsData + ); } else { currentPanel.webview.postMessage({ command: @@ -846,7 +894,7 @@ export class DataImport { } break; } - case "vscode-couchbase.tools.dataImport.getScopes":{ + case "vscode-couchbase.tools.dataImport.getScopes": { const scopes = await getScopes( message.bucketId, connection @@ -865,7 +913,7 @@ export class DataImport { }); break; } - case "vscode-couchbase.tools.dataImport.getDatasetFile":{ + case "vscode-couchbase.tools.dataImport.getDatasetFile": { const options: vscode.OpenDialogOptions = { canSelectMany: false, openLabel: "Choose Dataset File", @@ -886,7 +934,7 @@ export class DataImport { }); break; } - case "vscode-couchbase.tools.dataImport.getKeysBack":{ + case "vscode-couchbase.tools.dataImport.getKeysBack": { const datasetAndTargetData = message.datasetAndTargetData; const keysAndAdvancedSettingsData = @@ -895,11 +943,12 @@ export class DataImport { currentPanel.webview.html = await getDatasetAndCollection( bucketNameArr, - datasetAndTargetData + datasetAndTargetData, + keysAndAdvancedSettingsData ); break; } - case "vscode-couchbase.tools.dataImport.fetchKeyPreview":{ + case "vscode-couchbase.tools.dataImport.fetchKeyPreview": { const keyType = message.keyType; const keyExpr = message.keyExpr; const preview = await this.updateKeyPreview( @@ -914,10 +963,15 @@ export class DataImport { break; } case "vscode-couchbase.tools.dataImport.onBackSummary": { - const datasetAndCollectionData = message.datasetAndCollectionData; - const keysAndAdvancedSettingsData = message.keysAndAdvancedSettingsData; + const datasetAndCollectionData = + message.datasetAndCollectionData; + const keysAndAdvancedSettingsData = + message.keysAndAdvancedSettingsData; currentPanel.webview.html = getLoader("Data Import"); - currentPanel.webview.html = getKeysAndAdvancedSettings(datasetAndCollectionData); + currentPanel.webview.html = getKeysAndAdvancedSettings( + datasetAndCollectionData, + keysAndAdvancedSettingsData + ); break; } } diff --git a/src/tools/CBImport.ts b/src/tools/CBImport.ts index c626c5b1..f7012921 100644 --- a/src/tools/CBImport.ts +++ b/src/tools/CBImport.ts @@ -90,7 +90,7 @@ export class CBImport { cmd.push(importData.format); } if(importData.fileFormat === "csv"){ // Field Seperator is taken as ',' by default and only required in case of CSV File Format - cmd.push("--field-seperator"); + cmd.push("--field-separator"); cmd.push(","); cmd.push("--infer-types"); // Adding infer types flag as well for csv } diff --git a/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts b/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts index c02db6bd..91b34cf5 100644 --- a/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts +++ b/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts @@ -1,5 +1,9 @@ -export const getDatasetAndCollection = async (buckets: string[], prefilledData: any): Promise => { - return /*html*/ ` +export const getDatasetAndCollection = async ( + buckets: string[], + prefilledData: any, + keysAndAdvancedSettingsData: any +): Promise => { + return /*html*/ ` @@ -142,10 +146,10 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData: @@ -173,11 +177,11 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData: @@ -232,6 +236,9 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData: setInitialValue('collectionsDropdown', prefilledData.collectionsDropdown); setInitialValue('scopesDynamicField',prefilledData.scopesDynamicField); setInitialValue('collectionsDynamicField',prefilledData.collectionsDynamicField); + + // After the prefill, we can reset the prefilled data so that it does not interfere anymore + ${prefilledData = undefined}; } else { // We want scopeDetails for 1st bucket so calling the function on load of the webview const selectElement = document.getElementById('bucket'); @@ -317,7 +324,7 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData: break; case 'vscode-couchbase.tools.dataImport.datasetFile': const dataset = message.dataset; - document.getElementById("selectedFile").setAttribute("value", dataset); + document.getElementById("selectedFile").value = dataset; break; case "vscode-couchbase.tools.dataImport.getDatasetAndCollectionPageFormValidationError": const error = message.error; @@ -361,7 +368,10 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData: }; vscode.postMessage({ command: 'vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage', - data: formData + data: formData, + keysAndAdvancedSettingsData: ${JSON.stringify( + keysAndAdvancedSettingsData + )} }); } diff --git a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts index bc706617..b454ad86 100644 --- a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts +++ b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts @@ -1,5 +1,8 @@ -export const getKeysAndAdvancedSettings = (lastPageData: any): string => { - return /*html*/` +export const getKeysAndAdvancedSettings = ( + datasetAndCollectionData: any, + prefilledData: any +): string => { + return /*html*/ ` @@ -200,7 +203,7 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => {
Fetch preview
- +

@@ -263,6 +266,38 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { }); }); + async function setInitialValue(fieldId, value) { + const field = document.getElementById(fieldId); + + if (field && value && value.trim() !== "") { + field.value = value; + if(fieldId === "keyOptions") { + field.dispatchEvent(new Event('change')); + } + } + } + + window.onload = async function() { + let prefilledData = ${JSON.stringify(prefilledData)}; + if(prefilledData){ // If prefilled data is undefined, go with defaults only + setInitialValue('keyOptions', prefilledData.keyOptions); + setInitialValue('keyFieldName', prefilledData.keyFieldName); + setInitialValue('customExpression', prefilledData.customExpression); + setInitialValue('skipFirstDocuments', prefilledData.skipDocsOrRows); + setInitialValue('importUptoDocuments', prefilledData.limitDocsOrRows); + setInitialValue('ignoreFields', prefilledData.ignoreFields); + setInitialValue('threads', prefilledData.threads); + + // Verbose Log + if(prefilledData.verboseLog){ + document.getElementById('verboseLog').checked = true; + } + } + + // After setting prefilled data, set it to undefined so that it doesn't interfere + ${(prefilledData = undefined)} + } + function fetchKeyPreview() { var keyOptions = document.getElementById('keyOptions').value; @@ -312,13 +347,15 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { vscode.postMessage({ command: 'vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage', data: formData, - datasetAndCollectionData: ${JSON.stringify(lastPageData)} + datasetAndCollectionData: ${JSON.stringify( + datasetAndCollectionData + )} }); } function onBackClick(event) { event.preventDefault(); // prevent form submission - let lastPageData = ${JSON.stringify(lastPageData)}; + let lastPageData = ${JSON.stringify(datasetAndCollectionData)}; var keyOptions = document.getElementById('keyOptions').value; var keyFieldName = document.getElementById('keyFieldName').value; var customExpression = document.getElementById('customExpression').value; @@ -352,6 +389,9 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { switch (message.command) { case "vscode-couchbase.tools.dataImport.sendKeyPreview": let preview = message.preview; + if(preview === ""){ + preview = "No Preview to Show!"; + } document.getElementById("keyPreviewTextArea").innerHTML = preview; break; case "vscode-couchbase.tools.dataImport.getKeysAndAdvancedSettingsPageFormValidationError": @@ -363,4 +403,4 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => { `; -}; \ No newline at end of file +}; diff --git a/src/webViews/tools/dataImport/getSummary.webview.ts b/src/webViews/tools/dataImport/getSummary.webview.ts index 0f1b4ca9..b44f1e29 100644 --- a/src/webViews/tools/dataImport/getSummary.webview.ts +++ b/src/webViews/tools/dataImport/getSummary.webview.ts @@ -2,23 +2,24 @@ export const getSummary = async ( datasetAndCollectionData: any, keysAndAdvancedSettingsData: any ) => { - // Scope And Collection Data Processing for summary let scopeAndCollectionValue = ""; - let scopeAndCollectionExpr = datasetAndCollectionData.scopeCollectionExpression; - const scopeAndCollectionType = datasetAndCollectionData.scopesAndCollections; + let scopeAndCollectionExpr = + datasetAndCollectionData.scopeCollectionExpression; + const scopeAndCollectionType = + datasetAndCollectionData.scopesAndCollections; let showExtraScopeAndCollection = false; let scopeData = ""; let collectionData = ""; - - if(scopeAndCollectionType === "defaultCollection") { + + if (scopeAndCollectionType === "defaultCollection") { scopeAndCollectionValue = "Default Scope and Collection"; - } else if(scopeAndCollectionType === "SpecifiedCollection") { + } else if (scopeAndCollectionType === "SpecifiedCollection") { scopeAndCollectionValue = "Specifed Scope and Collection"; showExtraScopeAndCollection = true; scopeData = datasetAndCollectionData.scopesDropdown; collectionData = datasetAndCollectionData.collectionsDropdown; - } else if(scopeAndCollectionType === "dynamicCollection") { + } else if (scopeAndCollectionType === "dynamicCollection") { scopeAndCollectionValue = "Dynamic Scope and Collection"; showExtraScopeAndCollection = true; scopeData = datasetAndCollectionData.scopesDynamicField; @@ -32,13 +33,13 @@ export const getSummary = async ( let showExtraKeyField = false; const documentKeyType = keysAndAdvancedSettingsData.keyOptions; - if(documentKeyType === "random") { + if (documentKeyType === "random") { documentKeyValue = "Generate Random UUID for each Document"; - } else if(documentKeyType === "fieldValue") { + } else if (documentKeyType === "fieldValue") { documentKeyValue = "Use the value of field as the key"; showExtraKeyField = true; keyField = keysAndAdvancedSettingsData.keyFieldName; - } else if(documentKeyType === "customExpression") { + } else if (documentKeyType === "customExpression") { documentKeyValue = "Generate key based on custom expression"; } @@ -129,28 +130,35 @@ export const getSummary = async (

Import Data

+

Summary

Dataset Path:
-
${datasetAndCollectionData.dataset}
+
${ + datasetAndCollectionData.dataset + }
Bucket Name:
-
${datasetAndCollectionData.bucket}
+
${ + datasetAndCollectionData.bucket + }
Scopes and Collections:
${scopeAndCollectionValue}
- ${showExtraScopeAndCollection === true ? - `
+ ${ + showExtraScopeAndCollection === true + ? `
- Scope:
${scopeData}

- Collection:
${collectionData}
- ` : `` + ` + : `` }
- Scope and Collection Expression:
@@ -161,11 +169,12 @@ export const getSummary = async (
Document Key:
${documentKeyValue}
${ - showExtraKeyField === true ? - `
+ showExtraKeyField === true + ? `
- Key Field:
${keyField}
- ` : `` + ` + : `` }
- Key Expression:
@@ -175,41 +184,54 @@ export const getSummary = async (
Advanced Settings:
${ - keysAndAdvancedSettingsData.skipDocsOrRows && keysAndAdvancedSettingsData.skipDocsOrRows.trim() !== "" ? - ` + keysAndAdvancedSettingsData.skipDocsOrRows && + keysAndAdvancedSettingsData.skipDocsOrRows.trim() !== + "" + ? `
- Skip First Documents:
${keysAndAdvancedSettingsData.skipDocsOrRows}
- ` : `` + ` + : `` } ${ - keysAndAdvancedSettingsData.limitDocsOrRows && keysAndAdvancedSettingsData.limitDocsOrRows.trim() !== "" ? - ` + keysAndAdvancedSettingsData.limitDocsOrRows && + keysAndAdvancedSettingsData.limitDocsOrRows.trim() !== + "" + ? `
- Limit Documents/Rows:
${keysAndAdvancedSettingsData.limitDocsOrRows}
- ` : `` + ` + : `` } ${ - keysAndAdvancedSettingsData.ignoreFields && keysAndAdvancedSettingsData.ignoreFields.trim() !== "" ? - ` + keysAndAdvancedSettingsData.ignoreFields && + keysAndAdvancedSettingsData.ignoreFields.trim() !== + "" + ? `
- Ignore Fields:
${keysAndAdvancedSettingsData.ignoreFields}
- ` : `` + ` + : `` } ${ - keysAndAdvancedSettingsData.threads && keysAndAdvancedSettingsData.threads.trim() !== "" ? - ` + keysAndAdvancedSettingsData.threads && + keysAndAdvancedSettingsData.threads.trim() !== "" + ? `
- Threads:
${keysAndAdvancedSettingsData.threads}
- ` : `` + ` + : `` }
- Verbose Log:
-
${keysAndAdvancedSettingsData.verboseLog}
+
${ + keysAndAdvancedSettingsData.verboseLog + }
@@ -225,8 +247,12 @@ export const getSummary = async ( event.preventDefault(); vscode.postMessage({ command: "vscode-couchbase.tools.dataImport.runImport", - datasetAndCollectionData: ${JSON.stringify(datasetAndCollectionData)}, - keysAndAdvancedSettingsData: ${JSON.stringify(keysAndAdvancedSettingsData)} + datasetAndCollectionData: ${JSON.stringify( + datasetAndCollectionData + )}, + keysAndAdvancedSettingsData: ${JSON.stringify( + keysAndAdvancedSettingsData + )} }) } @@ -234,8 +260,12 @@ export const getSummary = async ( event.preventDefault(); vscode.postMessage({ command: "vscode-couchbase.tools.dataImport.onBackSummary", - datasetAndCollectionData: ${JSON.stringify(datasetAndCollectionData)}, - keysAndAdvancedSettingsData: ${JSON.stringify(keysAndAdvancedSettingsData)} + datasetAndCollectionData: ${JSON.stringify( + datasetAndCollectionData + )}, + keysAndAdvancedSettingsData: ${JSON.stringify( + keysAndAdvancedSettingsData + )} }) }