Skip to content

Commit

Permalink
HTML export service levels (datacontract#192)
Browse files Browse the repository at this point in the history 
* Added service level property to data contract specification.
* Add blocks for the servicelevel definitions
---------

Co-authored-by: Joachim Praetorius <[email protected]>
  • Loading branch information
@jeeger @jpraetorius
jeeger and jpraetorius authored May 13, 2024
1 parent a9b51e0 commit 82fbc87
Show file tree
Hide file tree
Showing 4 changed files with 345 additions and 2 deletions.
50 changes: 49 additions & 1 deletion datacontract/model/data_contract_specification.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import os
from typing import List, Dict
from typing import List, Dict, Optional

import pydantic as pyd
import yaml
Expand Down Expand Up @@ -113,6 +113,53 @@ class Quality(pyd.BaseModel):
type: str = None
specification: str | object = None

class Availability(pyd.BaseModel):
description: Optional[str] = None
percentage: Optional[str] = None

class Retention(pyd.BaseModel):
description: Optional[str] = None
period: Optional[str] = None
unlimited: Optional[bool] = None
timestampField: Optional[str] = None

class Latency(pyd.BaseModel):
description: Optional[str] = None
threshold: Optional[str] = None
sourceTimestampField: Optional[str] = None
processedTimestampField: Optional[str] = None

class Freshness(pyd.BaseModel):
description: Optional[str] = None
threshold: Optional[str] = None
timestampField: Optional[str] = None

class Frequency(pyd.BaseModel):
description: Optional[str] = None
type: Optional[str] = None
interval: Optional[str] = None
cron: Optional[str] = None

class Support(pyd.BaseModel):
description: Optional[str] = None
time: Optional[str] = None
responseTime: Optional[str] = None

class Backup(pyd.BaseModel):
description: Optional[str] = None
interval: Optional[str] = None
cron: Optional[str] = None
recoveryTime: Optional[str] = None
recoveryPoint: Optional[str] = None

class ServiceLevel(pyd.BaseModel):
availability: Optional[Availability] = None
retention: Optional[Retention] = None
latency: Optional[Latency] = None
freshness: Optional[Freshness] = None
frequency: Optional[Frequency] = None
support: Optional[Support] = None
backup: Optional[Backup] = None

class DataContractSpecification(pyd.BaseModel):
dataContractSpecification: str = None
Expand All @@ -125,6 +172,7 @@ class DataContractSpecification(pyd.BaseModel):
# schema: Dict[str, str]
examples: List[Example] = []
quality: Quality = None
servicelevels: Optional[ServiceLevel] = None

@classmethod
def from_file(cls, file):
Expand Down
232 changes: 231 additions & 1 deletion datacontract/templates/datacontract.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,237 @@ <h1 class="text-base font-semibold leading-6 text-gray-900">

{# TODO add definitions #}

{# TODO add service levels #}
{% if datacontract.servicelevels %}
<section id="servicelevels">
<div class="px-4 sm:px-0">
<h1 class="text-base font-semibold leading-6 text-gray-900">Service Levels</h1>
<p class="text-sm text-gray-500">Servicelevels of the data contract</p>
</div>
{% if datacontract.servicelevels.availability %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="availability">
Availability
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.availability.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Percentage</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.availability.percentage }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.retention %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="retention">
Retention
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.retention.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Period</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.retention.period }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Unlimited</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.retention.unlimited }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.latency %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="latency">
Latency
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.latency.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Threshold</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.latency.threshold }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Source Timestamp field</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.latency.sourceTimestampField
}}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Processed Timestamp field</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.latency.processedTimestampField
}}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.freshness %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="freshness">
Freshness
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.freshness.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Threshold</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.freshness.threshold }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Timestamp field</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.freshness.timestampField }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.frequency %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="frequency">
Frequency
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.frequency.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Type</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.frequency.type }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Interval</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.frequency.interval }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Cron</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.frequency.cron }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.support %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="support">
Support
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.support.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Time</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.support.time }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Response Time</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.support.responseTime }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
{% if datacontract.servicelevels.backup %}
<h2 class="mt-2 text-base font-semibold leading-6 text-gray-900" id="backup">
Backup
</h2>
<div class="mt-2 overflow-hidden shadow sm:rounded-lg bg-white">
<div class="px-4 py-5 sm:px-6">
<dl class="grid grid-cols-1 gap-x-4 gap-y-6 sm:grid-cols-2">
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Description</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.backup.description }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Interval</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.backup.interval }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Cron</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap font-mono">{{ datacontract.servicelevels.backup.cron }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Recovery Time</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.backup.recoveryTime }}</span>
</dd>
</div>
<div class="sm:col-span-1">
<dt class="text-sm font-medium text-gray-500">Recovery Point</dt>
<dd class="mt-1 text-sm text-gray-900">
<span class="whitespace-pre-wrap">{{ datacontract.servicelevels.backup.recoveryPoint }}</span>
</dd>
</div>
</dl>
</div>
</div>
{% endif %}
</section>
{% endif %}

{% if quality_specification %}
<section id="quality">
Expand Down
58 changes: 58 additions & 0 deletions tests/fixtures/examples/datacontract_servicelevels.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
dataContractSpecification: 0.9.2
id: "61111-0002"
info:
title: "Verbraucherpreisindex: Deutschland, Monate"
description: A data contract for the distribution and use of the German Consumer Price Index data.
version: 1.0.0
owner: my-domain-team
models:
verbraucherpreisindex:
description: Model representing the Consumer Price Index for Germany
fields:
wert:
description: Value of the Consumer Price Index
type: integer
required: true
jahrMonat:
description: Year and month of the data
type: string
required: true
qualitaet:
description: Quality of the data
type: string
enum:
- "vorlaeufig"
- "endgueltig"
verarbeitet:
description: Time when this dataset was processed, year and month
type: string

examples:
- type: json
description: Example entry for CPI data
model: verbraucherpreisindex
data: |-
[{ "wert": 99, "jahrMonat": "2022-00", "verarbeitet": "2022-01" },
{ "wert": 100, "jahrMonat": "2022-01", "verarbeitet": "2022-02" },
{ "wert": 101, "jahrMonat": "2022-02", "qualitaet": "vorlaeufig", "verarbeitet": "2022-03" }]
servicelevels:
availability:
percentage: "99.9%"
retention:
period: P5Y
timestampField: jahrMonat
latency:
threshold: P1M
sourceTimestampField: verarbeitet
freshness:
threshold: P1M
timestampField: verarbeitet
frequency:
type: streaming
support:
time: 24/7
responseTime: P1h
backup:
interval: daily
recoveryTime: 2 hours
7 changes: 7 additions & 0 deletions tests/test_test_examples_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,10 @@ def test_json():
print(run)
print(run.result)
assert run.result == "passed"

def test_with_service_level():
data_contract = DataContract(data_contract_file="fixtures/examples/datacontract_servicelevels.yaml", examples=True)
run = data_contract.test()
print(run)
print(run.result)
assert run.result == "passed"

0 comments on commit 82fbc87

Please sign in to comment.