Merge pull request #118 from amosproj/92-clean-up-task-technical-debt…

…-clean-up

92 clean up task technical debt clean up

apps/analyzer/metadata_analyzer/analyzer.py

@@ -1,3 +1,5 @@
+import datetime
+
 class Analyzer:
     def init(database, backend, simple_analyzer, simple_rule_based_analyzer):
         Analyzer.database = database
@@ -19,21 +21,38 @@ def analyze():
 
     # Convert a result from the database into the format used by the backend
     def _convert_result(result):
+        backup_type = {
+                "F": "FULL",
+                "I": "INCREMENTAL",
+                "D": "DIFFERENTIAL",
+                "C": "COPY"
+        }[result.fdi_type]
         return {
             "id": result.uuid,
             "sizeMB": result.data_size / 1_000_000,
             "creationDate": result.start_time.isoformat(),
+            "type": backup_type
         }
 
+    def _get_start_date(data, alert_type, backup_type):
+        latest_id = Analyzer.backend.get_latest_alert_id(alert_type, backup_type)
+        if latest_id == "":
+            return datetime.datetime.min
+        else:
+            latest_alerts = [result.start_time for result in data if result.uuid == latest_id]
+            assert len(latest_alerts) == 1
+            return latest_alerts[0]
+
+
     def update_data():
         results = list(Analyzer.database.get_results())
 
         # Batch the api calls to the backend for improved efficiency
         batch = []
         count = 0
         for result in results:
-            # Only send 'full' backups
-            if result.fdi_type != "F":
+            # Only send real backups
+            if result.is_backup <= 0:
                 continue
 
             # Only send backups where the relevant data is not null
@@ -56,15 +75,18 @@ def update_data():
 
     def simple_rule_based_analysis(alert_limit):
         data = list(Analyzer.database.get_results())
-        result = Analyzer.simple_rule_based_analyzer.analyze(data, alert_limit)
+        start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "FULL")
+        result = Analyzer.simple_rule_based_analyzer.analyze(data, alert_limit, start_date)
         return result
 
     def simple_rule_based_analysis_diff(alert_limit):
         data = list(Analyzer.database.get_results())
-        result = Analyzer.simple_rule_based_analyzer.analyze_diff(data,alert_limit)
+        start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "DIFFERENTIAL")
+        result = Analyzer.simple_rule_based_analyzer.analyze_diff(data, alert_limit, start_date)
         return result
 
     def simple_rule_based_analysis_inc(alert_limit):
         data = list(Analyzer.database.get_results())
-        result = Analyzer.simple_rule_based_analyzer.analyze_inc(data,alert_limit)
+        start_date = Analyzer._get_start_date(data, "SIZE_ALERT", "INCREMENTAL")
+        result = Analyzer.simple_rule_based_analyzer.analyze_inc(data, alert_limit, start_date)
         return result

apps/analyzer/metadata_analyzer/backend.py

@@ -9,7 +9,15 @@ def send_backup_data_batched(self, batch):
         r = requests.post(url, json=batch)
         r.raise_for_status()
 
-    def create_alert(self, alert):
+    def create_size_alert(self, alert):
         url = self.backend_url + "alerting/size"
         r = requests.post(url, json=alert)
         r.raise_for_status()
+
+    def get_latest_alert_id(self, alert_type, backup_type=None):
+        url = self.backend_url + f"alerting/type/{alert_type}/latest"
+        if backup_type != None:
+            url += f"?backupType={backup_type}"
+        r = requests.get(url)
+        r.raise_for_status()
+        return r.text

apps/analyzer/metadata_analyzer/creation_date_alert.py

@@ -0,0 +1,12 @@
+class CreationDateAlert:
+    def __init__(self, result, reference_date):
+        self.uuid = result.uuid
+        self.date = result.start_time
+        self.reference_date = reference_date
+
+    def as_json(self):
+        return {
+                "backupId": self.uuid,
+                "date": self.date.isoformat(),
+                "referenceDate": self.reference_date.isoformat()
+        }

apps/analyzer/metadata_analyzer/main.py

@@ -13,6 +13,7 @@
 swagger = Swagger(app)
 load_dotenv(dotenv_path=".env")
 
+
 @app.route("/")
 def hello_world():
     """Most basic example endpoint that returns a hello world message.
@@ -110,6 +111,7 @@ def analyze():
     """
     return jsonify(Analyzer.analyze())
 
+
 @app.route("/updateBasicBackupData", methods=["POST"])
 def update_data():
     """Updates the backend database with values taken from the analyzer database.
@@ -143,23 +145,153 @@ def update_data():
     """
     return jsonify(Analyzer.update_data())
 
+
 @app.route("/simpleRuleBasedAnalysis", methods=["POST"])
 def simple_rule_based_analysis():
-    json = request.get_json()
-    alert_limit = json["alertLimit"]
-    return jsonify(Analyzer.simple_rule_based_analysis(alert_limit))
+    """Fulfills a simple rule based analysis on full backups.
+    ---
+    parameters:
+      - name: Input
+        in: query
+        name: alertLimit
+        schema:
+            type: integer
+    definitions:
+        Alerts:
+            type: object
+            properties:
+                type:
+                    type: int
+                    example: 1
+                value:
+                    type: int
+                    example: 12345
+                referenceValue:
+                    type: int
+                    example: 12300
+                backupId:
+                    type: string
+        AlertLimit:
+            type: object
+            properties:
+                alertLimit:
+                    type: int
+                    example: 1
+    responses:
+        200:
+            description: Alerts for the analyzed data
+            schema:
+                $ref: '#/definitions/Alerts'
+        400:
+            description: The value set for the alert limit was not valid
+    """
+    alert_limit = request.args.get("alertLimit")
+
+    try:
+        int(alert_limit)
+        return jsonify(Analyzer.simple_rule_based_analysis(int(alert_limit)))
+    except ValueError:
+        print("Alert limit is not a number")
+        return "Invalid value for alert limit", 400
+
 
 @app.route("/simpleRuleBasedAnalysisDiff", methods=["POST"])
 def simple_rule_based_analysis_diff():
-    json = request.get_json()
-    alert_limit = json["alertLimit"]
-    return jsonify(Analyzer.simple_rule_based_analysis_diff(alert_limit))
+    """Fulfills a simple rule based analysis on diff backups.
+    ---
+    parameters:
+      - name: Input
+        in: query
+        name: alertLimit
+        schema:
+            type: integer
+    definitions:
+        Alerts:
+            type: object
+            properties:
+                type:
+                    type: int
+                    example: 1
+                value:
+                    type: int
+                    example: 12345
+                referenceValue:
+                    type: int
+                    example: 12300
+                backupId:
+                    type: string
+        AlertLimit:
+            type: object
+            properties:
+                alertLimit:
+                    type: int
+                    example: 1
+    responses:
+        200:
+            description: Alerts for the analyzed data
+            schema:
+                $ref: '#/definitions/Alerts'
+        400:
+            description: The value set for the alert limit was not valid
+    """
+    alert_limit = request.args.get("alertLimit")
+
+    try:
+        int(alert_limit)
+        return jsonify(Analyzer.simple_rule_based_analysis_diff(int(alert_limit)))
+    except ValueError:
+        print("Alert limit is not a number")
+        return "Invalid value for alert limit", 400
+
 
 @app.route("/simpleRuleBasedAnalysisInc", methods=["POST"])
 def simple_rule_based_analysis_inc():
-    json = request.get_json()
-    alert_limit = json["alertLimit"]
-    return jsonify(Analyzer.simple_rule_based_analysis_inc(alert_limit))
+    """Fulfills a simple rule based analysis on inc backups.
+    ---
+    parameters:
+      - name: Input
+        in: query
+        name: alertLimit
+        schema:
+            type: integer
+    definitions:
+        Alerts:
+            type: object
+            properties:
+                type:
+                    type: int
+                    example: 1
+                value:
+                    type: int
+                    example: 12345
+                referenceValue:
+                    type: int
+                    example: 12300
+                backupId:
+                    type: string
+        AlertLimit:
+            type: object
+            properties:
+                alertLimit:
+                    type: int
+                    example: 1
+    responses:
+        200:
+            description: Alerts for the analyzed data
+            schema:
+                $ref: '#/definitions/Alerts'
+        400:
+            description: The value set for the alert limit was not valid
+    """
+    alert_limit = request.args.get("alertLimit")
+
+    try:
+        int(alert_limit)
+        return jsonify(Analyzer.simple_rule_based_analysis_inc(int(alert_limit)))
+    except ValueError:
+        print("Alert limit is not a number")
+        return "Invalid value for alert limit", 400
+
 
 def main():
     database = Database()