Aavish Gilbert J - Feedback

.gitignore

@@ -0,0 +1,2 @@
+venv1
+__pycache__

AppscriptCode/GoogleScript.gs

@@ -0,0 +1,75 @@
+function onEditTrigger(e) {
+  // Check if the edited range contains data
+  if (!e || !e.range) return;
+
+  // Get the sheet and range information
+  var sheet = e.range.getSheet();
+  var range = e.range;
+  var sheetName = sheet.getName();
+  var rowStart = range.getRow();
+  var colStart = range.getColumn();
+  var numRows = range.getNumRows();
+  var numCols = range.getNumColumns();
+
+  var modifiedCells = [];
+
+  // Iterate over all modified cells
+  for (var i = 0; i < numRows; i++) {
+    for (var j = 0; j < numCols; j++) {
+      var cellRow = rowStart + i;
+      var cellCol = colStart + j;
+      var editedValue = sheet.getRange(cellRow, cellCol).getValue();
+      var oldValue = e.oldValue; // Get the old value before the edit
+
+      // Determine the type of modification (created, updated, deleted)
+      var action = '';
+      if (!oldValue && editedValue) {
+        action = 'created';
+      } else if (oldValue && !editedValue) {
+        action = 'deleted';
+      } else if (oldValue !== editedValue) {
+        action = 'updated';
+      }
+
+      if (action) {
+        // Collect information about the modified cell
+        modifiedCells.push({
+          sheetName: sheetName,
+          row: cellRow,
+          column: cellCol,
+          oldValue: oldValue || null,
+          newValue: editedValue || null,
+          action: action
+        });
+      }
+    }
+  }
+
+  // If any cells were modified, send the data
+  if (modifiedCells.length > 0) {
+    Logger.log(modifiedCells);
+    sendPostRequest(modifiedCells);
+  }
+}
+
+function sendPostRequest(modifiedCells) {
+  var url = 'https://example.com/your-backend-endpoint';  // Replace with your backend endpoint
+
+  var data = {
+    timestamp: new Date().toISOString(),
+    modifiedCells: modifiedCells
+  };
+
+  var options = {
+    'method': 'post',
+    'contentType': 'application/json',
+    'payload': JSON.stringify(data)
+  };
+
+  try {
+    var response = UrlFetchApp.fetch(url, options);
+    Logger.log('POST request sent successfully: ' + response.getContentText());
+  } catch (error) {
+    Logger.log('Error sending POST request: ' + error.toString());
+  }
+}

AppscriptCode/sheetToDb.gs

@@ -0,0 +1,101 @@
+function onEditTrigger(e) {
+  Logger.log("Triggered onEdit");
+
+  // Check if the edited range contains data
+  if (!e || !e.range) {
+    Logger.log("No range or event detected");
+    return;
+  }
+
+  // Get the sheet and range information
+  var sheet = e.range.getSheet();
+  var range = e.range;
+  var sheetName = sheet.getName();
+  var rowStart = range.getRow();
+  var colStart = range.getColumn();
+  var numRows = range.getNumRows();
+  var numCols = range.getNumColumns();
+
+  Logger.log("Sheet Name: " + sheetName);
+  Logger.log("Row Start: " + rowStart + ", Column Start: " + colStart);
+  Logger.log("Number of Rows: " + numRows + ", Number of Columns: " + numCols);
+
+  var modifiedCells = [];
+
+  // Iterate over all modified cells
+  for (var i = 0; i < numRows; i++) {
+    for (var j = 0; j < numCols; j++) {
+      var cellRow = rowStart + i;
+      var cellCol = colStart + j;
+      var editedValue = sheet.getRange(cellRow, cellCol).getValue();
+      var oldValue = e.oldValue; // Get the old value before the edit
+
+      Logger.log("Processing cell: (" + cellRow + ", " + cellCol + ")");
+      Logger.log("Old Value: " + oldValue + ", New Value: " + editedValue);
+
+      // Determine the type of modification (inserted, updated, deleted)
+      var action = '';
+      if (!oldValue && editedValue) {
+        action = 'insert';
+        Logger.log("Action: insert");
+      } else if (oldValue && !editedValue) {
+        action = 'delete'; // Set action to delete if the cell is cleared
+        Logger.log("Action: delete");
+      } else if (oldValue !== editedValue) {
+        action = 'update';
+        Logger.log("Action: update");
+      }
+
+      if (action) {
+        // Collect information about the modified cell
+        var cellModification = {
+          row: cellRow,
+          column: cellCol,
+          value: action === 'delete' ? null : editedValue, // If delete, set value to null
+          operation: action
+        };
+
+        Logger.log("Cell Modification: " + JSON.stringify(cellModification));
+
+        modifiedCells.push(cellModification);
+      }
+    }
+  }
+
+  // Build the final JSON object
+  if (modifiedCells.length > 0) {
+    var jsonPayload = {
+      sheet_name: sheetName,
+      cells: modifiedCells
+    };
+
+    Logger.log("JSON Payload: " + JSON.stringify(jsonPayload)); // Log the JSON object
+
+    sendPostRequest(jsonPayload); // Function to send the JSON payload to the server
+  } else {
+    Logger.log("No modifications detected");
+  }
+}
+
+// Function to send the JSON data via POST request to Kafka endpoint
+function sendPostRequest(payload) {
+  Logger.log("Publishing to Kafka");
+
+  var url = "https://unique-powerful-husky.ngrok-free.app/kafka-publish-endpoint"; // Kafka publishing endpoint
+
+  var options = {
+    method: "POST",
+    contentType: "application/json",
+    payload: JSON.stringify(payload)
+  };
+
+  try {
+    var response = UrlFetchApp.fetch(url, options);
+    Logger.log("Kafka Publish Response: " + response.getContentText());
+  } catch (error) {
+    Logger.log("Error in Kafka publish request: " + error.toString());
+  }
+}
+
+
+

README.md

@@ -1,5 +1,15 @@
+[![Review Assignment Due Date](https://classroom.github.com/assets/deadline-readme-button-22041afd0340ce965d47ae6ef1cefeee28c7c493a6346c4f15d667ab976d596c.svg)](https://classroom.github.com/a/AHFn7Vbn)
 # Superjoin Hiring Assignment
 
+### Planned Architecture
+
+[![](https://mermaid.ink/img/pako:eNqVVV1r2zAU_StCUEhpQt79MGiTrrAta1Zn7MUQFPnGFrElV5I3Qsl_n2QpjmyUkfkl4X6ce-7V0dUHpiIHnGBaEaWWjBSS1BlH5uss6EWIooK0BNAKfTiP_WbKmpZEE5SgL-nr94vrQfDnnOmNZEUBcnIfeBTwfC2UfoP3FpQ--04ZD2s-Nk1KJWt0WPAhFzZzAvcD2wuMTE27q5gqfzY50bARX8n-QAYcqOCqraFzuCh1hUcXMuDgwVegFCkgBhtzlYTnFSwE31eM6mvVfhGmpclOQf4GOZj18skkc6CaCR7gzuujeq-2bddEaG_MoAqDFXGpI6fbojvUbXeEoZN6iluTK6p2VM4eXmygEkj-WYp64Bk1tzqmP74tn0ZdxeXjSHcZgzp70LRcmGEWV09s7Rr_31o-Td1e7u7O3wzkr4Y5Yqtb5IW7EHXdckbJZYaDmzSbfQpkniB3ZWyZHj9As-BOjBHcC4wFdWEJWjulItcg0sJ7NqJhFE288H1bzjXmtHCK9il7IZHPmvdSDuie5Yu8fv_JeaT1GHFXqy816mDMewQYI98jvfXinvuGgjYC9lYxO6LgxgbOEk88dWdAxjJZv6YbNLitfvDnnGgLj_TAxZ8K8gI85NXaA933BM6yDjiMNoOnMUi_mct4Zs88bwTjRt123CklFdmxiuljlHYS3V-RoOgyi8Rd2WyRyOiaw1Ncg6wJy81r2K2ODOsSashwYv7mRB4ynPGTiSOtFqmphhMtW5hiR8w_njjZk0oZK5gLLeTKP6_2Z4qlaIvSR5z-AgPBZ3Q?type=png)](https://mermaid.live/edit#pako:eNqVVV1r2zAU_StCUEhpQt79MGiTrrAta1Zn7MUQFPnGFrElV5I3Qsl_n2QpjmyUkfkl4X6ce-7V0dUHpiIHnGBaEaWWjBSS1BlH5uss6EWIooK0BNAKfTiP_WbKmpZEE5SgL-nr94vrQfDnnOmNZEUBcnIfeBTwfC2UfoP3FpQ--04ZD2s-Nk1KJWt0WPAhFzZzAvcD2wuMTE27q5gqfzY50bARX8n-QAYcqOCqraFzuCh1hUcXMuDgwVegFCkgBhtzlYTnFSwE31eM6mvVfhGmpclOQf4GOZj18skkc6CaCR7gzuujeq-2bddEaG_MoAqDFXGpI6fbojvUbXeEoZN6iluTK6p2VM4eXmygEkj-WYp64Bk1tzqmP74tn0ZdxeXjSHcZgzp70LRcmGEWV09s7Rr_31o-Td1e7u7O3wzkr4Y5Yqtb5IW7EHXdckbJZYaDmzSbfQpkniB3ZWyZHj9As-BOjBHcC4wFdWEJWjulItcg0sJ7NqJhFE288H1bzjXmtHCK9il7IZHPmvdSDuie5Yu8fv_JeaT1GHFXqy816mDMewQYI98jvfXinvuGgjYC9lYxO6LgxgbOEk88dWdAxjJZv6YbNLitfvDnnGgLj_TAxZ8K8gI85NXaA933BM6yDjiMNoOnMUi_mct4Zs88bwTjRt123CklFdmxiuljlHYS3V-RoOgyi8Rd2WyRyOiaw1Ncg6wJy81r2K2ODOsSashwYv7mRB4ynPGTiSOtFqmphhMtW5hiR8w_njjZk0oZK5gLLeTKP6_2Z4qlaIvSR5z-AgPBZ3Q)
+
+
+### Video
+
+https://drive.google.com/file/d/18NcHJFeRSfQ9IOGx4iLDHjqH73XrxyQF/view?usp=sharing
+
 ### Welcome to Superjoin's hiring assignment! 🚀
 
 ### Objective
@@ -25,7 +35,7 @@ Many businesses use Google Sheets for collaborative data management and database
 - Ensure the solution can handle large datasets and high-frequency updates without performance degradation.
 - Optimize for scalability and efficiency.
 
-## Submission ⏰
+### Submission ⏰
 The timeline for this submission is: **Next 2 days**
 
 Some things you might want to take care of:
@@ -44,11 +54,11 @@ Once you're done, make sure you **record a video** showing your project working.
 
 We have a checklist at the bottom of this README file, which you should update as your progress with your assignment. It will help us evaluate your project.
 
-- [ ] My code's working just fine! 🥳
-- [ ] I have recorded a video showing it working and embedded it in the README ▶️
-- [ ] I have tested all the normal working cases 😎
-- [ ] I have even solved some edge cases (brownie points) 💪
-- [ ] I added my very planned-out approach to the problem at the end of this README 📜
+- [] My code's working just fine! 🥳
+- [] I have recorded a video showing it working and embedded it in the README ▶️
+- [] I have tested all the normal working cases 😎
+- [] I have even solved some edge cases (brownie points) 💪
+- [] I added my very planned-out approach to the problem at the end of this README 📜
 
 ## Got Questions❓
 Feel free to check the discussions tab, you might get some help there. Check out that tab before reaching out to us. Also, did you know, the internet is a great place to explore? 😛
@@ -58,4 +68,51 @@ We're available at [email protected] for all queries.
 All the best ✨.
 
 ## Developer's Section
-*Add your video here, and your approach to the problem (optional). Leave some comments for us here if you want, we will be reading this :)*
+# Google Sheets & MySQL Sync with Kafka Integration
+
+This project implements a real-time synchronization between Google Sheets and a MySQL database, leveraging Kafka for scalability and conflict resolution. The Flask server is hosted via Waitress and exposed to the internet using NGROK. Google Apps Script handles the detection of changes on Google Sheets, triggering the appropriate operations (Create, Read, Update, Delete) in real time.
+
+## Architecture Overview
+
+- **Flask Server**: Built using Python's Flask framework, served via Waitress.
+- **NGROK**: Used to expose the Flask server to the internet for handling requests from Google Sheets.
+- **Kafka**: Handles message passing for scalable synchronization between the Google Sheets and MySQL database. It also resolves conflicts when concurrent updates occur.
+- **Google Apps Script**: Detects changes in Google Sheets and triggers CRUD operations accordingly.
+- **MySQL Database**: Stores the data synced from the Google Sheets.
+
+## Key Features
+
+- **Real-Time Synchronization**: The system ensures that any changes made in Google Sheets (adding rows, updating values, etc.) are reflected in the MySQL database instantly.
+- **Scalability**: Kafka integration allows the system to scale horizontally, ensuring that multiple clients can interact with the database without performance degradation.
+- **Conflict Resolution**: Internal conflict-handling mechanisms resolve issues arising from concurrent edits.
+- **CRUD Operations**: Supports Create, Read, Update, and Delete operations seamlessly between Google Sheets and MySQL.
+
+## Challenges Faced
+
+One major roadblock encountered was configuring Kafka, as it was my first time working with it. The rest of the functionalities were more straightforward since I had prior experience implementing similar operations using Google Apps Script and Python during my internship at EKCS.
+
+## Personal Reflection
+
+Building this project was really fun and I learned a lot about Kafka in the process.
+
+---
+
+## How to Use
+
+1. **Setup MySQL Database**: Ensure you have a running MySQL instance.
+2. **Flask Server**: Start the Flask server using Waitress and expose it via NGROK.
+3. **Google Sheets**: Add the appropriate Apps Script to trigger updates on changes in Google Sheets.
+4. **Kafka**: Set up a Kafka broker and configure the consumer and producer in the Flask app.
+
+## Checklist
+- [YES] My code's working just fine! 🥳
+- [YES] I have recorded a video showing it working and embedded it in the README ▶️
+- [YES] I have tested all the normal working cases 😎
+- [YES] I have even solved some edge cases (brownie points) 💪
+- [YES] I added my very planned-out approach to the problem at the end of this README 📜
+
+## Author
+
+Aavish Gilbert J  
+Email: [email protected]  
+PES University, PES1UG21CS012