Skip to content

Commit

Permalink
datacontract catalog (datacontract#158)
Browse files Browse the repository at this point in the history
* First draft of catalog feature

Co-authored-by: jochen <[email protected]>
  • Loading branch information
simonharrer and jochenchrist authored Apr 25, 2024
1 parent 7fc43d1 commit ed6abcf
Show file tree
Hide file tree
Showing 10 changed files with 2,206 additions and 3 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

- Added test support for `azure` (#146)
- Added support for `delta` tables on S3 (#24)

- Added new command `datacontract catalog` that generates a data contract catalog with an `index.html` file.

## [0.10.1] - 2024-04-19

Expand Down
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ Commands
- [breaking](#breaking)
- [changelog](#changelog)
- [diff](#diff)
- [catalog](#catalog)

### init

Expand Down Expand Up @@ -725,6 +726,21 @@ Available import options:
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
```

### catalog

```
Usage: datacontract catalog [OPTIONS]
Create a html catalog of data contracts.
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│ --files TEXT Glob pattern for the data contract files to include in the catalog. [default: *.yaml] │
│ --output TEXT Output directory for the catalog html files. [default: catalog/] │
│ --help Show this message and exit. │
╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
```


## Integrations

Expand Down
70 changes: 70 additions & 0 deletions datacontract/catalog/catalog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
from dataclasses import dataclass

from jinja2 import PackageLoader, Environment, select_autoescape
import pytz
from datetime import datetime

from datacontract.export.html_export import get_version
from datacontract.data_contract import DataContract
from datacontract.model.data_contract_specification import DataContractSpecification


def create_data_contract_html(contracts, file, path):
data_contract = DataContract(data_contract_file=f"{file.absolute()}", inline_definitions=True)
html = data_contract.export(export_format="html")
spec = data_contract.get_data_contract_specification()
# html_filename = f"dc-{spec.id}-{spec.info.version}.html"
file_without_suffix = file.name.removesuffix(".yaml").removesuffix(".yml")
html_filename = f"{file_without_suffix}.html"
html_filepath = path / html_filename
with open(html_filepath, "w") as f:
f.write(html)
contracts.append(DataContractView(
html_filepath=html_filepath,
html_filename=html_filename,
spec=spec,
))
print(f"Created {html_filepath}")


@dataclass
class DataContractView:
"""Class for keeping track of an item in inventory."""
html_filepath: str
html_filename: str
spec: DataContractSpecification


def create_index_html(contracts, path):
index_filepath = path / "index.html"
with open(index_filepath, "w") as f:
# Load templates from templates folder
package_loader = PackageLoader("datacontract", "templates")
env = Environment(
loader=package_loader,
autoescape=select_autoescape(
enabled_extensions="html",
default_for_string=True,
),
)

# Load the required template
template = env.get_template("index.html")

style_content, _, _ = package_loader.get_source(env, "style/output.css")

tz = pytz.timezone('UTC')
now = datetime.now(tz)
formatted_date = now.strftime('%d %b %Y %H:%M:%S UTC')
datacontract_cli_version = get_version()

# Render the template with necessary data
html_string = template.render(
style=style_content,
formatted_date=formatted_date,
datacontract_cli_version=datacontract_cli_version,
contracts=contracts,
contracts_size=len(contracts),
)
f.write(html_string)
print(f"Created {index_filepath}")
26 changes: 25 additions & 1 deletion datacontract/cli.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
from enum import Enum
from importlib import metadata
from pathlib import Path
from typing import Iterable, Optional


import typer
from click import Context

from rich import box
from rich.console import Console
from rich.table import Table
from typer.core import TyperGroup
from typing_extensions import Annotated

from datacontract.catalog.catalog import create_index_html, create_data_contract_html
from datacontract.data_contract import DataContract
from datacontract.init.download_datacontract_file import download_datacontract_file, FileExistsException
from datacontract.init.download_datacontract_file import \
download_datacontract_file, FileExistsException

console = Console()

Expand Down Expand Up @@ -214,6 +219,25 @@ def import_(
console.print(result.to_yaml())


@app.command(name="catalog")
def catalog(
files: Annotated[Optional[str], typer.Option(help="Glob pattern for the data contract files to include in the catalog.")] = "*.yaml",
output: Annotated[Optional[str], typer.Option(help="Output directory for the catalog html files.")] = "catalog/",
):
"""
Create a html catalog of data contracts.
"""
path = Path(output)
path.mkdir(parents=True, exist_ok=True)
console.print(f"Created {output}")

contracts = []
for file in Path().glob(files):
create_data_contract_html(contracts, file, path)

create_index_html(contracts, path)


@app.command()
def breaking(
location_old: Annotated[str, typer.Argument(help="The location (url or path) of the old data contract yaml.")],
Expand Down
161 changes: 161 additions & 0 deletions datacontract/templates/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
<!doctype html>
<html class="h-full bg-gray-100" lang="en">
<head>
<title>Data Contract</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
{# <script src="https://cdn.tailwindcss.com"></script> #}
<style>
{{style | safe }}
</style>
</head>
<body class="h-full">

<div class="min-h-full flex flex-col">

<nav class="bg-white shadow-sm">
<div class="mx-auto max-w-7xl px-4 sm:px-6 lg:px-8">
<div class="flex h-16 justify-between">
<div class="flex">
<div class="flex flex-shrink-0 items-center mr-6">
<a class="text-xl text-gray-900 font-semibold" href="/">
Data Contract Catalog
</a>
</div>
</div>
<div class="hidden sm:ml-6 sm:flex sm:items-center">
<a href="https://datacontract.com" class="text-sky-500 hover:text-gray-700 text-sm font-semibold" target="_blank">datacontract.com</a>
</div>
</div>
</div>
</nav>

<main class="pb-7">

<div class="pt-5 mx-auto max-w-7xl sm:px-6 lg:px-8">
<div>
<div class="lg:flex lg:items-center lg:justify-between px-4 sm:px-0">
<div class="min-w-0 flex-1">
<h2 class="text-2xl font-bold leading-7 text-gray-900 sm:truncate sm:text-3xl sm:tracking-tight">
Data Contracts</h2>
<div class="mt-1 flex flex-col sm:mt-0 sm:flex-row sm:flex-wrap sm:space-x-6">
There are {{ contracts_size }} contracts in the catalog
</div>
</div>
</div>


</div>

<div>

<div class="space-y-6 mt-6">

<section id="catalog">


<div class="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div class="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
<div class="overflow-hidden shadow ring-1 ring-black ring-opacity-5 sm:rounded-lg">

<table class="min-w-full divide-y divide-gray-300">
<thead class="bg-gray-50">
<tr>
<th scope="col" class="py-3.5 pl-4 pr-3 text-left text-sm font-semibold text-gray-900 sm:pl-6">Title</th>
<th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">ID</th>
<th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">Version</th>
<th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">Owner</th>
</tr>
</thead>
<tbody class="divide-y divide-gray-200 bg-white">
{% for contract in contracts %}
<tr>
<td class="whitespace-nowrap py-4 pl-4 pr-3 text-sm font-medium text-gray-900 sm:pl-6">
<a class="text-indigo-600 hover:text-indigo-900" href="{{contract.html_filename}}">{{contract.spec.info.title}}</a>
</td>
<td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{{contract.spec.id}}
</td>
<td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{{contract.spec.info.version}}
</td>
<td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500">
{{contract.spec.info.owner}}
</td>
</tr>
{% endfor %}
</tbody>
</table>

</div>
</div>
</div>

</section>

</div>

</div>

<div class="mt-6 text-sm text-gray-400">
Created at {{formatted_date}} with <a href="https://cli.datacontract.com" class="text-gray-400 hover:text-gray-500">Data Contract CLI</a> v{{datacontract_cli_version}}
</div>

</div>
</main>

<dialog id="dialog-datacontract-yaml" class="relative z-10" aria-labelledby="modal-title" aria-modal="true">
<div class="fixed inset-0 bg-gray-500 bg-opacity-75 transition-opacity"></div>

<div class="fixed inset-0 z-10 w-screen overflow-y-auto">
<div class="flex min-h-full items-end justify-center p-4 text-center sm:items-center sm:p-0">
<div class="relative transform rounded-lg bg-white px-4 pb-4 pt-5 text-left shadow-xl transition-all sm:my-8 sm:w-4/5 sm:p-6">
<div class="absolute right-0 top-0 pr-4 pt-4">
<button type="button" onclick="document.getElementById('dialog-datacontract-yaml').close()" class="rounded-md bg-white text-gray-400 hover:text-gray-500">
<span class="sr-only">Close</span>
<svg class="h-6 w-6" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true">
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
<div>
<div class="mt-3 mb-3 text-center sm:mt-5">

<h3 class="text-base font-semibold leading-6 text-gray-900" id="modal-title">Data Contract YAML</h3>
</div>
<pre class="overflow-x-auto text-sm bg-gray-50 p-4"><code>{{datacontract_yaml}}</code></pre>
</div>
<div class="mt-5 sm:mt-6">
<button type="button" onclick="document.getElementById('dialog-datacontract-yaml').close()"
class="inline-flex w-full justify-center rounded-md bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600">Close</button>
</div>
</div>
</div>
</div>
</dialog>


<footer class="bg-white mt-auto text-sm text-gray-400">
<div class="mx-auto max-w-7xl py-5 px-6 md:flex md:items-center md:justify-between lg:px-8">
<div class="flex justify-center space-x-6 md:order-2">
<div>
<a href="https://cli.datacontract.com" class="text-gray-400 hover:text-gray-500 px-2">Data Contract CLI</a>

<a href="https://datacontract.com" class="text-gray-400 hover:text-gray-500 px-2">Data Contract Specification</a>
</div>


</div>
<div class="mt-8 md:order-1 md:mt-0">
<p class="text-center leading-5 text-gray-400">
Supported with ❤️ by <a href="https://datamesh-manager.com" class="text-gray-400 hover:text-gray-500">Data Mesh Manager</a>
</p>
</div>
</div>

</footer>

</div>

</body>
</html>
25 changes: 25 additions & 0 deletions datacontract/templates/style/output.css
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,21 @@ video {
padding-bottom: 0.5rem;
}

.py-3 {
padding-top: 0.75rem;
padding-bottom: 0.75rem;
}

.py-3\.5 {
padding-top: 0.875rem;
padding-bottom: 0.875rem;
}

.py-4 {
padding-top: 1rem;
padding-bottom: 1rem;
}

.py-5 {
padding-top: 1.25rem;
padding-bottom: 1.25rem;
Expand Down Expand Up @@ -1054,6 +1069,11 @@ video {
color: rgb(17 24 39 / var(--tw-text-opacity));
}

.text-indigo-600 {
--tw-text-opacity: 1;
color: rgb(79 70 229 / var(--tw-text-opacity));
}

.text-sky-500 {
--tw-text-opacity: 1;
color: rgb(14 165 233 / var(--tw-text-opacity));
Expand Down Expand Up @@ -1146,6 +1166,11 @@ video {
color: rgb(55 65 81 / var(--tw-text-opacity));
}

.hover\:text-indigo-900:hover {
--tw-text-opacity: 1;
color: rgb(49 46 129 / var(--tw-text-opacity));
}

.focus-visible\:outline:focus-visible {
outline-style: solid;
}
Expand Down
2 changes: 1 addition & 1 deletion datacontract/templates/style/tailwind.config.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module.exports = {
content: ["../datacontract.html"],
content: ["../datacontract.html", "../index.html"],
theme: { },
plugins: [],
}
Loading

0 comments on commit ed6abcf

Please sign in to comment.