Skip to content

Commit

Permalink
fixed csv import issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@lokesh-couchbase
lokesh-couchbase committed Oct 27, 2023
1 parent 471aff4 commit a0df063
Show file tree
Hide file tree
Showing 5 changed files with 215 additions and 81 deletions.
114 changes: 84 additions & 30 deletions src/commands/tools/dataImport.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ interface IDataImportWebviewState {
export class DataImport {
cachedJsonDocs: string[] = [];
cachedCsvDocs: Map<string, string[]> = new Map();
PREVIEW_SIZE: number = 6; // Define your desired preview size
PREVIEW_SIZE: number = 6;
JSON_FILE_EXTENSION: string = ".json";
CSV_FILE_EXTENSION: string = ".csv";
JSON_FILE_FORMAT: string = "json";
Expand Down Expand Up @@ -96,7 +96,6 @@ export class DataImport {
if (counter === 0 && !insideArray) {
if (!line.trim().startsWith("[")) {
logger.debug("Not a JSON array");
// TODO: Give error to user as well
readStream.close();
return;
}
Expand Down Expand Up @@ -128,8 +127,49 @@ export class DataImport {
readStream.on("end", () => {
readStream.close();
});
} else {
const headers = await this.sampleElementFromCsvFile(
datasetPath,
1
);
if (headers === null) {
return;
}
for (
let lineNumber = 2;
lineNumber < 2 + this.PREVIEW_SIZE;
lineNumber++
) {
const data = await this.sampleElementFromCsvFile(
datasetPath,
lineNumber
);
if (data === null) {
continue;
}

for (
let headersIndex = 0;
headersIndex < headers.length;
headersIndex++
) {
let cachedCsv = this.cachedCsvDocs.get(
headers[headersIndex]
);
if (!cachedCsv) {
cachedCsv = new Array(this.PREVIEW_SIZE);
}
cachedCsv[lineNumber - 2] = data[headersIndex];
this.cachedCsvDocs.set(
headers[headersIndex],
cachedCsv
);
}
}
}
} catch (err) {}
} catch (err) {
logger.error(err);
}
};

async updateKeyPreview(keyType: string, keyExpr: string): Promise<string> {
Expand All @@ -140,6 +180,10 @@ export class DataImport {
this.cachedCsvDocs.size === 0)
) {
await this.readAndProcessPartialDataFromDataset();
// Wait 500ms extra to cache everything
await new Promise((resolve) => {
setTimeout(resolve, 500);
});
}

const previewContent: string[] = [];
Expand Down Expand Up @@ -675,14 +719,6 @@ export class DataImport {
);
}

if (
formData.ignoreFields &&
String(formData.ignoreFields).trim() !== ""
) {
// Ignore fields exists, validating it
// TODO: Add validations for CSV File format
}

if (
!formData.threads ||
!formData.threads.trim() ||
Expand Down Expand Up @@ -774,35 +810,40 @@ export class DataImport {
try {
currentPanel.webview.html = await getDatasetAndCollection(
bucketNameArr,
undefined,
undefined
);
currentPanel.webview.onDidReceiveMessage(async (message) => {
switch (message.command) {
// ADD cases here :)
case "vscode-couchbase.tools.dataImport.runImport": {
const keysAndAdvancedSettingsData = message.keysAndAdvancedSettingsData;
const keysAndAdvancedSettingsData =
message.keysAndAdvancedSettingsData;
const datasetAndCollectionData =
message.datasetAndCollectionData;

CBImport.import({
bucket: datasetAndCollectionData.bucket, // TODO: bucket should be taken from other form
bucket: datasetAndCollectionData.bucket,
dataset: datasetAndCollectionData.dataset,
fileFormat: this.fileFormat,
format: this.format,
scopeCollectionExpression:
datasetAndCollectionData.scopeCollectionExpression,
generateKeyExpression:
keysAndAdvancedSettingsData.generateKeyExpression,
skipDocsOrRows: keysAndAdvancedSettingsData.skipDocsOrRows,
limitDocsOrRows: keysAndAdvancedSettingsData.limitDocsOrRows,
ignoreFields: keysAndAdvancedSettingsData.ignoreFields,
keysAndAdvancedSettingsData.generateKeyExpression,
skipDocsOrRows:
keysAndAdvancedSettingsData.skipDocsOrRows,
limitDocsOrRows:
keysAndAdvancedSettingsData.limitDocsOrRows,
ignoreFields:
keysAndAdvancedSettingsData.ignoreFields,
threads: keysAndAdvancedSettingsData.threads,
verbose: keysAndAdvancedSettingsData.verboseLog,
});

break;
}
case "vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage":{
case "vscode-couchbase.tools.dataImport.nextGetKeysAndAdvancedSettingsPage": {
const keysAndAdvancedSettingsformData = message.data;
const datasetAndCollectionData =
message.datasetAndCollectionData;
Expand All @@ -814,8 +855,10 @@ export class DataImport {
// Go to summary page
currentPanel.webview.html =
getLoader("Data Import");
currentPanel.webview.html = await getSummary(datasetAndCollectionData, keysAndAdvancedSettingsformData);

currentPanel.webview.html = await getSummary(
datasetAndCollectionData,
keysAndAdvancedSettingsformData
);
} else {
currentPanel.webview.postMessage({
command:
Expand All @@ -825,8 +868,10 @@ export class DataImport {
}
break;
}
case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage":{
case "vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage": {
let formData = message.data;
const keysAndAdvancedSettingsData =
message.keysAndAdvancedSettingsData;
const validationError =
await this.validateDatasetAndCollectionFormData(
formData
Expand All @@ -836,7 +881,10 @@ export class DataImport {
currentPanel.webview.html =
getLoader("Data Import");
currentPanel.webview.html =
getKeysAndAdvancedSettings(formData);
getKeysAndAdvancedSettings(
formData,
keysAndAdvancedSettingsData
);
} else {
currentPanel.webview.postMessage({
command:
Expand All @@ -846,7 +894,7 @@ export class DataImport {
}
break;
}
case "vscode-couchbase.tools.dataImport.getScopes":{
case "vscode-couchbase.tools.dataImport.getScopes": {
const scopes = await getScopes(
message.bucketId,
connection
Expand All @@ -865,7 +913,7 @@ export class DataImport {
});
break;
}
case "vscode-couchbase.tools.dataImport.getDatasetFile":{
case "vscode-couchbase.tools.dataImport.getDatasetFile": {
const options: vscode.OpenDialogOptions = {
canSelectMany: false,
openLabel: "Choose Dataset File",
Expand All @@ -886,7 +934,7 @@ export class DataImport {
});
break;
}
case "vscode-couchbase.tools.dataImport.getKeysBack":{
case "vscode-couchbase.tools.dataImport.getKeysBack": {
const datasetAndTargetData =
message.datasetAndTargetData;
const keysAndAdvancedSettingsData =
Expand All @@ -895,11 +943,12 @@ export class DataImport {
currentPanel.webview.html =
await getDatasetAndCollection(
bucketNameArr,
datasetAndTargetData
datasetAndTargetData,
keysAndAdvancedSettingsData
);
break;
}
case "vscode-couchbase.tools.dataImport.fetchKeyPreview":{
case "vscode-couchbase.tools.dataImport.fetchKeyPreview": {
const keyType = message.keyType;
const keyExpr = message.keyExpr;
const preview = await this.updateKeyPreview(
Expand All @@ -914,10 +963,15 @@ export class DataImport {
break;
}
case "vscode-couchbase.tools.dataImport.onBackSummary": {
const datasetAndCollectionData = message.datasetAndCollectionData;
const keysAndAdvancedSettingsData = message.keysAndAdvancedSettingsData;
const datasetAndCollectionData =
message.datasetAndCollectionData;
const keysAndAdvancedSettingsData =
message.keysAndAdvancedSettingsData;
currentPanel.webview.html = getLoader("Data Import");
currentPanel.webview.html = getKeysAndAdvancedSettings(datasetAndCollectionData);
currentPanel.webview.html = getKeysAndAdvancedSettings(
datasetAndCollectionData,
keysAndAdvancedSettingsData
);
break;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/tools/CBImport.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ export class CBImport {
cmd.push(importData.format);
}
if(importData.fileFormat === "csv"){ // Field Seperator is taken as ',' by default and only required in case of CSV File Format
cmd.push("--field-seperator");
cmd.push("--field-separator");
cmd.push(",");
cmd.push("--infer-types"); // Adding infer types flag as well for csv
}
Expand Down
28 changes: 19 additions & 9 deletions src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
export const getDatasetAndCollection = async (buckets: string[], prefilledData: any): Promise<string> => {
return /*html*/ `
export const getDatasetAndCollection = async (
buckets: string[],
prefilledData: any,
keysAndAdvancedSettingsData: any
): Promise<string> => {
return /*html*/ `
<!DOCTYPE html>
<html lang="en">
<head>
Expand Down Expand Up @@ -142,10 +146,10 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
<label for="bucket">Bucket:</label>
<select name="bucket" id="bucket" onchange="onBucketClick(value)">
${buckets.map((bucketName, index) => {
return `
return `
<option value="${bucketName}" ${
index === 0 && "selected"
}>${bucketName}</option>
index === 0 && "selected"
}>${bucketName}</option>
`;
})}
</select>
Expand Down Expand Up @@ -173,11 +177,11 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
<!-- Dynamic Scopes and Collections -->
<div id="dynamicCollectionContainer" hidden>
<label for="scopesDynamicField">Scope Field:</label>
<input type="text" name="scopesDynamicField" id="scopesDynamicField" value="%cbms%">
<input type="text" name="scopesDynamicField" id="scopesDynamicField" value="" placeholder="%cbms%">
<br>
<label for="collectionsDynamicField">Collection Field:</label>
<input type="text" name="collectionsDynamicField" id="collectionsDynamicField" value="%cbmc%">
<input type="text" name="collectionsDynamicField" id="collectionsDynamicField" value="" placeholder="%cbmc%">
<br>
</div>
Expand Down Expand Up @@ -232,6 +236,9 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
setInitialValue('collectionsDropdown', prefilledData.collectionsDropdown);
setInitialValue('scopesDynamicField',prefilledData.scopesDynamicField);
setInitialValue('collectionsDynamicField',prefilledData.collectionsDynamicField);
// After the prefill, we can reset the prefilled data so that it does not interfere anymore
${prefilledData = undefined};
} else {
// We want scopeDetails for 1st bucket so calling the function on load of the webview
const selectElement = document.getElementById('bucket');
Expand Down Expand Up @@ -317,7 +324,7 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
break;
case 'vscode-couchbase.tools.dataImport.datasetFile':
const dataset = message.dataset;
document.getElementById("selectedFile").setAttribute("value", dataset);
document.getElementById("selectedFile").value = dataset;
break;
case "vscode-couchbase.tools.dataImport.getDatasetAndCollectionPageFormValidationError":
const error = message.error;
Expand Down Expand Up @@ -361,7 +368,10 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
};
vscode.postMessage({
command: 'vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage',
data: formData
data: formData,
keysAndAdvancedSettingsData: ${JSON.stringify(
keysAndAdvancedSettingsData
)}
});
}
Expand Down
Loading

0 comments on commit a0df063

Please sign in to comment.