end to end import works

Couchbase-Ecosystem · Oct 25, 2023 · 1dfb694 · 1dfb694
1 parent 419aca8
commit 1dfb694
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 28 deletions.
diff --git a/src/commands/tools/dataImport.ts b/src/commands/tools/dataImport.ts
@@ -32,6 +32,7 @@ export class DataImport {
   JSON_FILE_FORMAT: string = "json";
   CSV_FILE_FORMAT: string = "csv";
   datasetField: string = "";
+  format: string = "";
   readonly UUID_FLAG = "#UUID#";
   readonly MONO_INCR_FLAG = "#MONO_INCR#";
   readonly WORDS_WITH_PERCENT_SYMBOLS_REGEX = "%(\\w+)%";
@@ -76,7 +77,6 @@ export class DataImport {
   private readAndProcessPartialDataFromDataset = async () => {
     try {
       const datasetPath: string = this.datasetField;
-      console.log("datasetPath",datasetPath);
       if (datasetPath.endsWith(this.JSON_FILE_EXTENSION)) {
         const readStream = fs.createReadStream(datasetPath, {
           encoding: "utf8",
@@ -163,10 +163,9 @@ export class DataImport {
             let matches: string[] = [];
 
             let match;
-            console.log("before while");
+
             while ((match = pattern.exec(expression)) !== null) {
                 // match[1] contains the captured word (the part between % symbols)
-                console.log(match);
                 if(match[1] === ""){
                   break;
                 }
@@ -176,8 +175,6 @@ export class DataImport {
             matches.forEach(match => {
                 fieldNamesList.push(match.replace(/%/g, ''));
             });
-            console.log("matches", matches);
-            console.log("fieldNamesList", fieldNamesList);
 
 
             for (let i = 0; i < Math.min(this.cachedJsonDocs.length, this.PREVIEW_SIZE); i++) {
@@ -227,7 +224,6 @@ export class DataImport {
             }
         }
     }
-    console.log(previewContent);
     return previewContent.join("\n");
 }
 
@@ -258,7 +254,7 @@ export class DataImport {
         resolve(result);
       });
       pipeline.on("error", (err) => {
-        console.error(err);
+        logger.error(err);
         resolve(null);
       });
     });
@@ -286,7 +282,7 @@ export class DataImport {
           }
         })
         .on("error", (err) => {
-          console.error(err);
+          logger.error(err);
           resolve([]);
         });
     });
@@ -322,7 +318,7 @@ export class DataImport {
           }
         }
       } catch (e) {
-        console.error(e);
+        logger.error(e);
         return false;
       }
     }
@@ -337,11 +333,11 @@ export class DataImport {
       const startBuffer = await this.readFirstTwoNonEmptyCharacters(filePath);
       const endBuffer = await this.readLastTwoNonEmptyCharacters(filePath);
 
-      const firstChar = startBuffer[1];
-      const secondChar = startBuffer[0];
+      const firstChar = startBuffer[0];
+      const secondChar = startBuffer[1];
 
-      const secondLastChar = endBuffer[1];
-      const lastChar = endBuffer[0];
+      const secondLastChar = endBuffer[0];
+      const lastChar = endBuffer[1];
 
       if (
         firstChar === "[" &&
@@ -453,14 +449,17 @@ export class DataImport {
     }
 
     if (this.fileFormat === this.JSON_FILE_FORMAT) {
-      let format: string | null = await this.detectDatasetFormat(
+      let currentFormat: string | null = await this.detectDatasetFormat(
         datasetFilePath
       );
-      console.log(format);
-      if (format) {
-        console.log(`Detected format: ${format}`);
+
+      if (currentFormat) {
+        this.format = currentFormat;
+        console.log(currentFormat);
+        logger.info("detected format " + currentFormat);
       } else {
-        console.log("Format not detected.");
+        this.format = "";
+        logger.error("format not detected");
         errors.push("Please enter valid json file format only");
       }
 
@@ -620,14 +619,12 @@ export class DataImport {
             const runFormData = message.data;
             const datasetAndCollectionData = message.datasetAndCollectionData;
             const runValidationError = await this.validateFormData(runFormData);
-            console.log("all data at end");
-            console.log(runFormData);
-            console.log(datasetAndCollectionData);
             if (runValidationError === "" || true) {
               CBImport.import({
                 bucket: datasetAndCollectionData.bucket, // TODO: bucket should be taken from other form
                 dataset: datasetAndCollectionData.dataset,
-                fileFormat: "json",
+                fileFormat: this.fileFormat,
+                format: this.format,
                 scopeCollectionExpression:
                   datasetAndCollectionData.scopeCollectionExpression,
                 generateKeyExpression: runFormData.generateKeyExpression,
@@ -695,7 +692,6 @@ export class DataImport {
             );
             break;
           case "vscode-couchbase.tools.dataImport.fetchKeyPreview":
-            console.log("inside data import");
             const keyType = message.keyType;
             const keyExpr = message.keyExpr;
             const preview = await this.updateKeyPreview(keyType, keyExpr);

diff --git a/src/tools/CBImport.ts b/src/tools/CBImport.ts
@@ -17,6 +17,7 @@ export interface ICBImportData {
     ignoreFields: string | undefined;
     threads: number;
     verbose: boolean;
+    format: string;
 }
 
 export class CBImport {
@@ -84,11 +85,11 @@ export class CBImport {
         cmd.push("--dataset");
         cmd.push(`\"file://${importData.dataset}\"`);
 
-        if(importData.fileFormat === "JSON"){ // If JSON File Format, Lines or Arrays needs to be specified as file type
+        if(importData.fileFormat === "json"){ // If JSON File Format, Lines or Arrays needs to be specified as file type
             cmd.push("--format");
-            cmd.push(importData.fileFormat);
+            cmd.push(importData.format);
         }
-        if(importData.fileFormat === "CSV"){ // Field Seperator is taken as ',' by default and only required in case of CSV File Format
+        if(importData.fileFormat === "csv"){ // Field Seperator is taken as ',' by default and only required in case of CSV File Format
             cmd.push("--field-seperator");
             cmd.push(",");
             cmd.push("--infer-types"); // Adding infer types flag as well for csv

diff --git a/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts b/src/webViews/tools/dataImport/getDatasetAndCollection.webview.ts
@@ -334,14 +334,17 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
             const bucket = document.getElementById('bucket').value;
             const scopesAndCollections = document.getElementById('scopesAndCollections').value;
             let scopesDropdown, collectionsDropdown, scopesDynamicField, collectionsDynamicField;
+            let scopeCollectionExpression = "_default._default"; // Default case of default values
 
             // Check which option is selected in the "Scopes And Collections" dropdown
             if (scopesAndCollections === 'SpecifiedCollection') {
                 scopesDropdown = document.getElementById('scopesDropdown').value;
                 collectionsDropdown = document.getElementById('collectionsDropdown').value;
+                scopeCollectionExpression = collectionsDropdown; //The value of collections dropdown already has scope data
             } else if (scopesAndCollections === 'dynamicCollection') {
                 scopesDynamicField = document.getElementById('scopesDynamicField').value;
                 collectionsDynamicField = document.getElementById('collectionsDynamicField').value;
+                scopeCollectionExpression = scopesDynamicField + '.' + collectionsDynamicField;
             }
 
             // Consolidate data
@@ -353,7 +356,8 @@ export const getDatasetAndCollection = async (buckets: string[], prefilledData:
                 collectionsDropdown,
                 scopesDynamicField,
                 collectionsDynamicField,
-                scopesSpecData
+                scopesSpecData,
+                scopeCollectionExpression
             };
             vscode.postMessage({
                 command: 'vscode-couchbase.tools.dataImport.nextGetDatasetAndCollectionPage',

diff --git a/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts b/src/webViews/tools/dataImport/getKeysAndAdvancedSettings.webview.ts
@@ -287,6 +287,13 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => {
                 var ignoreFields = document.getElementById('ignoreFields').value;
                 var threads = document.getElementById('threads').value;
                 var verboseLog = document.getElementById('verboseLog').checked;
+                let generateKeyExpression = "#UUID#"; // Default for setting key expression based on UUID
+                if(keyOptions === "fieldValue"){
+                    generateKeyExpression = "%" + keyFieldName + "%";
+                } else if (keyOptions === "customExpression"){
+                    generateKeyExpression = customExpression;
+                }
+                
 
                 // Consolidate data
                 var formData = {
@@ -297,7 +304,8 @@ export const getKeysAndAdvancedSettings = (lastPageData: any): string => {
                     limitDocsOrRows: limitDocsOrRows,
                     ignoreFields: ignoreFields,
                     threads: threads,
-                    verboseLog: verboseLog
+                    verboseLog: verboseLog,
+                    generateKeyExpression: generateKeyExpression
                 };
 
                 vscode.postMessage({