implemented changelog (datacontract#87)

implemented changelog

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added export format **protobuf**: `datacontract export --format protobuf`
 - Added export format **terraform**: `datacontract export --format terraform`
 - Added extensive linting on data contracts. `datacontract lint` will now check for a variety of possible errors in the data contract, such as missing descriptions, incorrect references to models or fields, nonsensical constraints, and more.
+- Added changelog command: `datacontract changelog` will now generate a changelog based on the changes in the data contract. This will be useful for keeping track of changes in the data contract over time. 
 
 ## [0.9.6-2] - 2024-03-04
 

README.md

@@ -83,7 +83,10 @@ $ datacontract test --examples datacontract.yaml
 # find differences between to data contracts (Coming Soon)
 $ datacontract diff datacontract-v1.yaml datacontract-v2.yaml
 
-# fail pipeline on breaking changes  (Coming Soon)
+# find differences between to data contracts categorized into error, warning, and info.
+$ datacontract changelog datacontract-v1.yaml datacontract-v2.yaml
+
+# fail pipeline on breaking changes. Uses changelog internally and showing only error and warning.
 $ datacontract breaking datacontract-v1.yaml datacontract-v2.yaml
 
 # export model as jsonschema (other formats: avro, dbt, dbt-sources, dbt-staging-sql, jsonschema, odcs, rdf, sql (coming soon), sodacl, terraform)

datacontract/breaking/breaking.py

@@ -1,12 +1,13 @@
 from datacontract.breaking.breaking_rules import BreakingRules
-from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Location
+from datacontract.model.breaking_change import BreakingChanges, BreakingChange, Location, Severity
 from datacontract.model.data_contract_specification import Field, Model, Quality
 
 
 def quality_breaking_changes(
     old_quality: Quality,
     new_quality: Quality,
-    new_path: str
+    new_path: str,
+    include_severities: [Severity],
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
@@ -15,7 +16,7 @@ def quality_breaking_changes(
         severity = _get_rule(rule_name)
         description = "added quality"
 
-        if severity != "info":
+        if severity in include_severities:
             results.append(
                 BreakingChange(
                     description=description,
@@ -30,7 +31,7 @@ def quality_breaking_changes(
         severity = _get_rule(rule_name)
         description = "removed quality"
 
-        if severity != "info":
+        if severity in include_severities:
             results.append(
                 BreakingChange(
                     description=description,
@@ -47,7 +48,7 @@ def quality_breaking_changes(
             severity = _get_rule(rule_name)
             description = f"changed from `{old_quality.type}` to `{new_quality.type}`"
 
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description=description,
@@ -62,7 +63,7 @@ def quality_breaking_changes(
             rule_name = "quality_specification_updated"
             severity = _get_rule(rule_name)
             description = f"changed from `{old_quality.specification}` to `{new_quality.specification}`"
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description=description,
@@ -79,7 +80,8 @@ def quality_breaking_changes(
 def models_breaking_changes(
     old_models: dict[str, Model],
     new_models: dict[str, Model],
-    new_path: str
+    new_path: str,
+    include_severities: [Severity],
 ) -> list[BreakingChange]:
     composition = ["models"]
     results = list[BreakingChange]()
@@ -88,7 +90,7 @@ def models_breaking_changes(
         if model_name not in old_models.keys():
             rule_name = "model_added"
             severity = _get_rule(rule_name)
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description="added the model",
@@ -103,7 +105,7 @@ def models_breaking_changes(
         if model_name not in new_models.keys():
             rule_name = "model_removed"
             severity = _get_rule(rule_name)
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description="removed the model",
@@ -120,7 +122,8 @@ def models_breaking_changes(
                 old_model=old_model,
                 new_model=new_models[model_name],
                 new_path=new_path,
-                composition=composition + [model_name]
+                composition=composition + [model_name],
+                include_severities=include_severities,
             ))
 
     return results
@@ -130,7 +133,8 @@ def model_breaking_changes(
     old_model: Model,
     new_model: Model,
     new_path: str,
-    composition: list[str]
+    composition: list[str],
+    include_severities: [Severity]
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
@@ -160,7 +164,7 @@ def model_breaking_changes(
 
         if rule_name is not None:
             severity = _get_rule(rule_name)
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description=description,
@@ -176,7 +180,8 @@ def model_breaking_changes(
             old_fields=old_model.fields,
             new_fields=new_model.fields,
             new_path=new_path,
-            composition=composition + ["fields"]
+            composition=composition + ["fields"],
+            include_severities=include_severities,
         ))
 
     return results
@@ -186,15 +191,16 @@ def fields_breaking_changes(
     old_fields: dict[str, Field],
     new_fields: dict[str, Field],
     new_path: str,
-    composition: list[str]
+    composition: list[str],
+    include_severities: [Severity]
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
     for field_name, new_field in new_fields.items():
         if field_name not in old_fields.keys():
             rule_name = "field_added"
             severity = _get_rule(rule_name)
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description="added the field",
@@ -209,7 +215,7 @@ def fields_breaking_changes(
         if field_name not in new_fields.keys():
             rule_name = "field_removed"
             severity = _get_rule(rule_name)
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description='removed the field',
@@ -227,6 +233,7 @@ def fields_breaking_changes(
                 new_field=new_fields[field_name],
                 composition=composition + [field_name],
                 new_path=new_path,
+                include_severities=include_severities,
             ))
     return results
 
@@ -236,6 +243,7 @@ def field_breaking_changes(
     new_field: Field,
     composition: list[str],
     new_path: str,
+    include_severities: [Severity],
 ) -> list[BreakingChange]:
     results = list[BreakingChange]()
 
@@ -253,7 +261,8 @@ def field_breaking_changes(
                     old_fields=old_field.fields,
                     new_fields=new_field.fields,
                     new_path=new_path,
-                    composition=composition + [field_definition_field]
+                    composition=composition + [field_definition_field],
+                    include_severities=include_severities,
                 )
             )
             continue
@@ -290,7 +299,7 @@ def field_breaking_changes(
         if rule_name is not None:
             severity = _get_rule(rule_name)
             field_schema_name = "$ref" if field_definition_field == "ref" else field_definition_field
-            if severity != "info":
+            if severity in include_severities:
                 results.append(
                     BreakingChange(
                         description=description,
@@ -304,13 +313,13 @@ def field_breaking_changes(
     return results
 
 
-def _get_rule(rule_name):
+def _get_rule(rule_name) -> Severity:
     try:
         return getattr(BreakingRules, rule_name)
     except AttributeError:
         print(f'WARNING: Breaking Rule not found for {rule_name}!')
-        return 'error'
+        return Severity.ERROR
 
-def _camel_to_snake(s):
-    return ''.join(['_'+c.lower() if c.isupper() else c for c in s]).lstrip('_')
 
+def _camel_to_snake(s):
+    return ''.join(['_' + c.lower() if c.isupper() else c for c in s]).lstrip('_')

datacontract/breaking/breaking_rules.py

@@ -1,91 +1,94 @@
+from datacontract.model.breaking_change import Severity
+
+
 class BreakingRules:
     # model rules
-    model_added = 'info'
-    model_removed = 'error'
+    model_added = Severity.INFO
+    model_removed = Severity.ERROR
 
-    model_description_added = 'info'
-    model_description_removed = 'info'
-    model_description_updated = 'info'
+    model_description_added = Severity.INFO
+    model_description_removed = Severity.INFO
+    model_description_updated = Severity.INFO
 
-    model_type_updated = 'error'
+    model_type_updated = Severity.ERROR
 
     # field rules
-    field_added = 'info'
-    field_removed = 'error'
+    field_added = Severity.INFO
+    field_removed = Severity.ERROR
 
-    field_ref_added = 'warning'
-    field_ref_removed = 'warning'
-    field_ref_updated = 'warning'
+    field_ref_added = Severity.WARNING
+    field_ref_removed = Severity.WARNING
+    field_ref_updated = Severity.WARNING
 
-    field_type_added = 'warning'
-    field_type_removed = 'warning'
-    field_type_updated = 'error'
+    field_type_added = Severity.WARNING
+    field_type_removed = Severity.WARNING
+    field_type_updated = Severity.ERROR
 
-    field_format_added = 'warning'
-    field_format_removed = 'warning'
-    field_format_updated = 'error'
+    field_format_added = Severity.WARNING
+    field_format_removed = Severity.WARNING
+    field_format_updated = Severity.ERROR
 
-    field_required_updated = 'error'
+    field_required_updated = Severity.ERROR
 
-    field_primary_updated = 'warning'
+    field_primary_updated = Severity.WARNING
 
-    field_references_added = 'warning'
-    field_references_removed = 'warning'
-    field_references_updated = 'warning'
+    field_references_added = Severity.WARNING
+    field_references_removed = Severity.WARNING
+    field_references_updated = Severity.WARNING
 
-    field_unique_updated = 'error'
+    field_unique_updated = Severity.ERROR
 
-    field_description_added = 'info'
-    field_description_removed = 'info'
-    field_description_updated = 'info'
+    field_description_added = Severity.INFO
+    field_description_removed = Severity.INFO
+    field_description_updated = Severity.INFO
 
-    field_pii_added = 'warning'
-    field_pii_removed = 'error'
-    field_pii_updated = 'error'
+    field_pii_added = Severity.WARNING
+    field_pii_removed = Severity.ERROR
+    field_pii_updated = Severity.ERROR
 
-    field_classification_added = 'warning'
-    field_classification_removed = 'error'
-    field_classification_updated = 'error'
+    field_classification_added = Severity.WARNING
+    field_classification_removed = Severity.ERROR
+    field_classification_updated = Severity.ERROR
 
-    field_pattern_added = 'warning'
-    field_pattern_removed = 'error'
-    field_pattern_updated = 'error'
+    field_pattern_added = Severity.WARNING
+    field_pattern_removed = Severity.ERROR
+    field_pattern_updated = Severity.ERROR
 
-    field_min_length_added = 'warning'
-    field_min_length_removed = 'warning'
-    field_min_length_updated = 'error'
+    field_min_length_added = Severity.WARNING
+    field_min_length_removed = Severity.WARNING
+    field_min_length_updated = Severity.ERROR
 
-    field_max_length_added = 'warning'
-    field_max_length_removed = 'warning'
-    field_max_length_updated = 'error'
+    field_max_length_added = Severity.WARNING
+    field_max_length_removed = Severity.WARNING
+    field_max_length_updated = Severity.ERROR
 
-    field_minimum_added = 'warning'
-    field_minimum_removed = 'warning'
-    field_minimum_updated = 'error'
+    field_minimum_added = Severity.WARNING
+    field_minimum_removed = Severity.WARNING
+    field_minimum_updated = Severity.ERROR
 
-    field_exclusive_minimum_added = 'warning'
-    field_exclusive_minimum_removed = 'warning'
-    field_exclusive_minimum_updated = 'error'
+    field_exclusive_minimum_added = Severity.WARNING
+    field_exclusive_minimum_removed = Severity.WARNING
+    field_exclusive_minimum_updated = Severity.ERROR
 
-    field_maximum_added = 'warning'
-    field_maximum_removed = 'warning'
-    field_maximum_updated = 'error'
+    field_maximum_added = Severity.WARNING
+    field_maximum_removed = Severity.WARNING
+    field_maximum_updated = Severity.ERROR
 
-    field_exclusive_maximum_added = 'warning'
-    field_exclusive_maximum_removed = 'warning'
-    field_exclusive_maximum_updated = 'error'
+    field_exclusive_maximum_added = Severity.WARNING
+    field_exclusive_maximum_removed = Severity.WARNING
+    field_exclusive_maximum_updated = Severity.ERROR
 
-    field_enum_added = 'warning'
-    field_enum_removed = 'info'
-    field_enum_updated = 'error'
+    field_enum_added = Severity.WARNING
+    field_enum_removed = Severity.INFO
+    field_enum_updated = Severity.ERROR
 
-    field_tags_added = 'info'
-    field_tags_removed = 'info'
-    field_tags_updated = 'info'
+    field_tags_added = Severity.INFO
+    field_tags_removed = Severity.INFO
+    field_tags_updated = Severity.INFO
 
     # quality Rules
-    quality_added = 'info'
-    quality_removed = 'warning'
+    quality_added = Severity.INFO
+    quality_removed = Severity.WARNING
 
-    quality_type_updated = 'warning'
-    quality_specification_updated = 'warning'
+    quality_type_updated = Severity.WARNING
+    quality_specification_updated = Severity.WARNING