diff --git a/CHANGELOG.md b/CHANGELOG.md index fa889e6c..4fc25f9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added test support for `azure` (#146) - Added support for `delta` tables on S3 (#24) - +- Added new command `datacontract catalog` that generates a data contract catalog with an `index.html` file. ## [0.10.1] - 2024-04-19 diff --git a/README.md b/README.md index 0d427226..3b12f673 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ Commands - [breaking](#breaking) - [changelog](#changelog) - [diff](#diff) +- [catalog](#catalog) ### init @@ -725,6 +726,21 @@ Available import options: ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` +### catalog + +``` + + Usage: datacontract catalog [OPTIONS] + + Create a html catalog of data contracts. + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --files TEXT Glob pattern for the data contract files to include in the catalog. [default: *.yaml] │ +│ --output TEXT Output directory for the catalog html files. [default: catalog/] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + ## Integrations diff --git a/datacontract/catalog/catalog.py b/datacontract/catalog/catalog.py new file mode 100644 index 00000000..74703e60 --- /dev/null +++ b/datacontract/catalog/catalog.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass + +from jinja2 import PackageLoader, Environment, select_autoescape +import pytz +from datetime import datetime + +from datacontract.export.html_export import get_version +from datacontract.data_contract import DataContract +from datacontract.model.data_contract_specification import DataContractSpecification + + +def create_data_contract_html(contracts, file, path): + data_contract = DataContract(data_contract_file=f"{file.absolute()}", inline_definitions=True) + html = data_contract.export(export_format="html") + spec = data_contract.get_data_contract_specification() + # html_filename = f"dc-{spec.id}-{spec.info.version}.html" + file_without_suffix = file.name.removesuffix(".yaml").removesuffix(".yml") + html_filename = f"{file_without_suffix}.html" + html_filepath = path / html_filename + with open(html_filepath, "w") as f: + f.write(html) + contracts.append(DataContractView( + html_filepath=html_filepath, + html_filename=html_filename, + spec=spec, + )) + print(f"Created {html_filepath}") + + +@dataclass +class DataContractView: + """Class for keeping track of an item in inventory.""" + html_filepath: str + html_filename: str + spec: DataContractSpecification + + +def create_index_html(contracts, path): + index_filepath = path / "index.html" + with open(index_filepath, "w") as f: + # Load templates from templates folder + package_loader = PackageLoader("datacontract", "templates") + env = Environment( + loader=package_loader, + autoescape=select_autoescape( + enabled_extensions="html", + default_for_string=True, + ), + ) + + # Load the required template + template = env.get_template("index.html") + + style_content, _, _ = package_loader.get_source(env, "style/output.css") + + tz = pytz.timezone('UTC') + now = datetime.now(tz) + formatted_date = now.strftime('%d %b %Y %H:%M:%S UTC') + datacontract_cli_version = get_version() + + # Render the template with necessary data + html_string = template.render( + style=style_content, + formatted_date=formatted_date, + datacontract_cli_version=datacontract_cli_version, + contracts=contracts, + contracts_size=len(contracts), + ) + f.write(html_string) + print(f"Created {index_filepath}") \ No newline at end of file diff --git a/datacontract/cli.py b/datacontract/cli.py index 7173399a..6233508f 100644 --- a/datacontract/cli.py +++ b/datacontract/cli.py @@ -1,17 +1,22 @@ from enum import Enum from importlib import metadata +from pathlib import Path from typing import Iterable, Optional + import typer from click import Context + from rich import box from rich.console import Console from rich.table import Table from typer.core import TyperGroup from typing_extensions import Annotated +from datacontract.catalog.catalog import create_index_html, create_data_contract_html from datacontract.data_contract import DataContract -from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException +from datacontract.init.download_datacontract_file import \ + download_datacontract_file, FileExistsException console = Console() @@ -214,6 +219,25 @@ def import_( console.print(result.to_yaml()) +@app.command(name="catalog") +def catalog( + files: Annotated[Optional[str], typer.Option(help="Glob pattern for the data contract files to include in the catalog.")] = "*.yaml", + output: Annotated[Optional[str], typer.Option(help="Output directory for the catalog html files.")] = "catalog/", +): + """ + Create a html catalog of data contracts. + """ + path = Path(output) + path.mkdir(parents=True, exist_ok=True) + console.print(f"Created {output}") + + contracts = [] + for file in Path().glob(files): + create_data_contract_html(contracts, file, path) + + create_index_html(contracts, path) + + @app.command() def breaking( location_old: Annotated[str, typer.Argument(help="The location (url or path) of the old data contract yaml.")], diff --git a/datacontract/templates/index.html b/datacontract/templates/index.html new file mode 100644 index 00000000..b4b89fff --- /dev/null +++ b/datacontract/templates/index.html @@ -0,0 +1,161 @@ + + + + Data Contract + + + {# #} + + + + +
+ + + +
+ +
+
+
+
+

+ Data Contracts

+
+ There are {{ contracts_size }} contracts in the catalog +
+
+
+ + +
+ +
+ +
+ +
+ + +
+
+
+ + + + + + + + + + + + {% for contract in contracts %} + + + + + + + {% endfor %} + +
TitleIDVersionOwner
+ {{contract.spec.info.title}} + + {{contract.spec.id}} + + {{contract.spec.info.version}} + + {{contract.spec.info.owner}} +
+ +
+
+
+ +
+ +
+ +
+ +
+ Created at {{formatted_date}} with Data Contract CLI v{{datacontract_cli_version}} +
+ +
+
+ + +
+ +
+
+
+
+ +
+
+
+ +

Data Contract YAML

+
+ 
{{datacontract_yaml}}
+
+
+ +
+
+
+
+ + + + + +
+ + + diff --git a/datacontract/templates/style/output.css b/datacontract/templates/style/output.css index 103c77d7..d468b2dd 100644 --- a/datacontract/templates/style/output.css +++ b/datacontract/templates/style/output.css @@ -936,6 +936,21 @@ video { padding-bottom: 0.5rem; } +.py-3 { + padding-top: 0.75rem; + padding-bottom: 0.75rem; +} + +.py-3\.5 { + padding-top: 0.875rem; + padding-bottom: 0.875rem; +} + +.py-4 { + padding-top: 1rem; + padding-bottom: 1rem; +} + .py-5 { padding-top: 1.25rem; padding-bottom: 1.25rem; @@ -1054,6 +1069,11 @@ video { color: rgb(17 24 39 / var(--tw-text-opacity)); } +.text-indigo-600 { + --tw-text-opacity: 1; + color: rgb(79 70 229 / var(--tw-text-opacity)); +} + .text-sky-500 { --tw-text-opacity: 1; color: rgb(14 165 233 / var(--tw-text-opacity)); @@ -1146,6 +1166,11 @@ video { color: rgb(55 65 81 / var(--tw-text-opacity)); } +.hover\:text-indigo-900:hover { + --tw-text-opacity: 1; + color: rgb(49 46 129 / var(--tw-text-opacity)); +} + .focus-visible\:outline:focus-visible { outline-style: solid; } diff --git a/datacontract/templates/style/tailwind.config.js b/datacontract/templates/style/tailwind.config.js index dda4b59b..1aa627c6 100644 --- a/datacontract/templates/style/tailwind.config.js +++ b/datacontract/templates/style/tailwind.config.js @@ -1,5 +1,5 @@ module.exports = { - content: ["../datacontract.html"], + content: ["../datacontract.html", "../index.html"], theme: { }, plugins: [], } \ No newline at end of file diff --git a/tests/datacontract-1.html b/tests/datacontract-1.html new file mode 100644 index 00000000..6e0c4674 --- /dev/null +++ b/tests/datacontract-1.html @@ -0,0 +1,1841 @@ + + + + Data Contract + + + + + + + +
+ + + +
+ +
+
+
+
+

+ Data Contract

+
+ orders-unit-test +
+
+
+ +
+
+ + +
+ +
+ +
+ + +
+
+

Info

+

Information about the data contract

+
+
+ +
+ +
+
+
Title
+
Orders Unit Test
+
+ +
+
Version
+
1.0.0
+
+ + +
+
Description
+
+ The orders data contract +
+
+ + + +
+
Owner
+
+ checkout +
+
+ + + +
+
Contact
+
+ + + team-orders@example.com + + + + +
+
+ + +
+
+
+
+ + + +
+
+

Servers

+

Servers of the data contract

+
+ +
    + + +
  • +
    + +
    + + +
    + +
    + + + + + + + + + + + +
    + +
    + + + + + + + + + +
    + +
    + + + +
    + +
    + + + + + + + + + + +
    • + + +
+ +
+ + + + +
+
+

Terms

+

Terms and conditions of the data contract

+
+
+ +
+ +
+
+
Usage
+
+ This data contract serves to demo datacontract CLI export. +
+
+ +
+
Limitations
+
+ Not intended to use in production +
+
+ + +
+
Billing
+
+ free +
+
+ + + +
+
Notice Period
+
+ P3M +
+
+ + +
+
+
+
+ + + +
+
+
+

+ Data Model +

+

The logical data model

+
+
+ + + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ orders + table +
The orders model
+
+
+ order_id + +
+ 		+ + R + + + U + + + + varchar + + +
No description
+ +
+
+ order_total + +
+ 		+ + R + + + + + bigint + + +
The order_total field
+ +
+
+ order_status + +
+ 		+ + R + + + + + text + + +
No description
+ +
+
+
+
+
+ + +
+ + + + + + + +
+ +
+ +
+ Created at 25 Apr 2024 14:38:16 UTC with Data Contract CLI v0.10.1 +
+ +
+
+ + +
+ +
+
+
+
+ +
+
+
+ +

Data Contract YAML

+
+ 
dataContractSpecification: 0.9.2
+id: orders-unit-test
+info:
+  title: Orders Unit Test
+  version: 1.0.0
+  description: The orders data contract
+  owner: checkout
+  contact:
+    url: https://wiki.example.com/teams/checkout
+    email: team-orders@example.com
+servers:
+  production:
+    type: snowflake
+    account: my-account
+    database: my-database
+    schema_: my-schema
+terms:
+  usage: This data contract serves to demo datacontract CLI export.
+  limitations: Not intended to use in production
+  billing: free
+  noticePeriod: P3M
+models:
+  orders:
+    description: The orders model
+    type: table
+    fields:
+      order_id:
+        type: varchar
+        required: true
+        primary: false
+        unique: true
+        pii: true
+        classification: sensitive
+        pattern: ^B[0-9]+$
+        minLength: 8
+        maxLength: 10
+        tags:
+        - order_id
+      order_total:
+        type: bigint
+        required: true
+        primary: false
+        unique: false
+        description: The order_total field
+        minimum: 0
+        maximum: 1000000
+      order_status:
+        type: text
+        required: true
+        primary: false
+        unique: false
+        enum:
+        - pending
+        - shipped
+        - delivered
+
+
+
+ +
+
+
+
+ + + + + +
+ + + \ No newline at end of file diff --git a/tests/fixtures/catalog/datacontract-1.yaml b/tests/fixtures/catalog/datacontract-1.yaml new file mode 100644 index 00000000..95a10389 --- /dev/null +++ b/tests/fixtures/catalog/datacontract-1.yaml @@ -0,0 +1,49 @@ +dataContractSpecification: 0.9.2 +id: orders-unit-test +info: + title: Orders Unit Test + version: 1.0.0 + owner: checkout + description: The orders data contract + contact: + email: team-orders@example.com + url: https://wiki.example.com/teams/checkout +terms: + usage: This data contract serves to demo datacontract CLI export. + limitations: Not intended to use in production + billing: free + noticePeriod: P3M +servers: + production: + type: snowflake + account: my-account + database: my-database + schema: my-schema +models: + orders: + description: The orders model + fields: + order_id: + type: varchar + unique: true + required: true + minLength: 8 + maxLength: 10 + pii: true + classification: sensitive + tags: + - order_id + pattern: ^B[0-9]+$ + order_total: + type: bigint + required: true + description: The order_total field + minimum: 0 + maximum: 1000000 + order_status: + type: text + required: true + enum: + - pending + - shipped + - delivered \ No newline at end of file diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 00000000..6191c08f --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,17 @@ +import logging +import os +from pathlib import PosixPath + +from typer.testing import CliRunner + +from datacontract.cli import app + +logging.basicConfig(level=logging.DEBUG, force=True) + + +def test_cli(tmp_path: PosixPath): + runner = CliRunner() + result = runner.invoke(app, ["catalog", "--files", "fixtures/catalog/*.yaml", "--output", tmp_path]) + assert result.exit_code == 0 + assert os.path.exists(tmp_path / "index.html") + assert os.path.exists(tmp_path / "datacontract-1.html") \ No newline at end of file