Skip to content

Commit

Permalink
UPDATE
Browse files Browse the repository at this point in the history
  • Loading branch information
simonharrer committed Apr 29, 2024
1 parent 54e0635 commit 758d974
Show file tree
Hide file tree
Showing 5 changed files with 17 additions and 12 deletions.
2 changes: 1 addition & 1 deletion datacontract/catalog/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@


def create_data_contract_html(contracts, file: Path, path: Path):
data_contract = DataContract(data_contract_file=f"{file.absolute()}", inline_definitions=True)
data_contract = DataContract(data_contract_file=f"{file.absolute()}", inline_definitions=True, inline_quality=True)
html = data_contract.export(export_format="html")
spec = data_contract.get_data_contract_specification()
file_without_suffix = file.with_suffix(".html")
Expand Down
6 changes: 5 additions & 1 deletion datacontract/data_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def __init__(
publish_to_opentelemetry: bool = False,
spark: SparkSession = None,
inline_definitions: bool = False,
inline_quality: bool = False,
):
self._data_contract_file = data_contract_file
self._data_contract_str = data_contract_str
Expand All @@ -67,6 +68,7 @@ def __init__(
self._publish_to_opentelemetry = publish_to_opentelemetry
self._spark = spark
self._inline_definitions = inline_definitions
self._inline_quality = inline_quality
self.all_linters = {
ExampleModelLinter(),
QualityUsesSchemaLinter(),
Expand Down Expand Up @@ -95,6 +97,7 @@ def lint(self, enabled_linters: typing.Union[str, set[str]] = "all") -> Run:
self._data_contract,
self._schema_location,
inline_definitions=True,
inline_quality=True,
)
run.checks.append(
Check(type="lint", result="passed", name="Data contract is syntactically valid", engine="datacontract")
Expand Down Expand Up @@ -263,11 +266,12 @@ def get_data_contract_specification(self) -> DataContractSpecification:
data_contract=self._data_contract,
schema_location=self._schema_location,
inline_definitions=self._inline_definitions,
inline_quality=self._inline_quality
)

def export(self, export_format, model: str = "all", rdf_base: str = None, sql_server_type: str = "auto") -> str:
data_contract = resolve.resolve_data_contract(
self._data_contract_file, self._data_contract_str, self._data_contract, inline_definitions=True
self._data_contract_file, self._data_contract_str, self._data_contract, inline_definitions=True, inline_quality=True
)
if export_format == "jsonschema":
if data_contract.models is None:
Expand Down
13 changes: 7 additions & 6 deletions datacontract/lint/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ def resolve_data_contract(
data_contract: DataContractSpecification = None,
schema_location: str = None,
inline_definitions: bool = False,
inline_quality: bool = False,
) -> DataContractSpecification:
if data_contract_location is not None:
return resolve_data_contract_from_location(data_contract_location, schema_location, inline_definitions)
return resolve_data_contract_from_location(data_contract_location, schema_location, inline_definitions, inline_quality)
elif data_contract_str is not None:
return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions)
return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)
elif data_contract is not None:
return data_contract
else:
Expand All @@ -36,13 +37,13 @@ def resolve_data_contract(


def resolve_data_contract_from_location(
location, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = True
location, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
) -> DataContractSpecification:
if location.startswith("http://") or location.startswith("https://"):
data_contract_str = fetch_resource(location)
else:
data_contract_str = read_file(location)
return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, include_quality)
return resolve_data_contract_from_str(data_contract_str, schema_location, inline_definitions, inline_quality)


def inline_definitions_into_data_contract(spec: DataContractSpecification):
Expand Down Expand Up @@ -116,7 +117,7 @@ def get_quality_ref_file(quality_spec: str | object) -> str | object:


def resolve_data_contract_from_str(
data_contract_str, schema_location: str = None, inline_definitions: bool = False, include_quality: bool = False
data_contract_str, schema_location: str = None, inline_definitions: bool = False, inline_quality: bool = False
) -> DataContractSpecification:
data_contract_yaml_dict = to_yaml(data_contract_str)
validate(data_contract_yaml_dict, schema_location)
Expand All @@ -125,7 +126,7 @@ def resolve_data_contract_from_str(

if inline_definitions:
inline_definitions_into_data_contract(spec)
if spec.quality and include_quality:
if spec.quality and inline_quality:
resolve_quality_ref(spec.quality)

return spec
Expand Down
6 changes: 3 additions & 3 deletions tests/test_export_great_expectations.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,14 @@ def data_contract_complex() -> DataContractSpecification:
@pytest.fixture
def data_contract_great_expectations() -> DataContractSpecification:
return resolve.resolve_data_contract_from_location(
"./fixtures/great-expectations/datacontract.yaml", include_quality=True
"./fixtures/great-expectations/datacontract.yaml", inline_quality=True
)


@pytest.fixture
def data_contract_great_expectations_quality_file() -> DataContractSpecification:
return resolve.resolve_data_contract_from_location(
"./fixtures/great-expectations/datacontract_quality_file.yaml", include_quality=True
"./fixtures/great-expectations/datacontract_quality_file.yaml", inline_quality=True
)


Expand Down Expand Up @@ -263,7 +263,7 @@ def test_to_great_expectation_missing_quality_json_file():
"""
try:
resolve.resolve_data_contract_from_location(
"./fixtures/great-expectations/datacontract_missing_quality_file.yaml", include_quality=True
"./fixtures/great-expectations/datacontract_missing_quality_file.yaml", inline_quality=True
)
assert False
except DataContractException as dataContractException:
Expand Down
2 changes: 1 addition & 1 deletion tests/test_export_sodacl.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def test_to_sodacl(check_expected: str):

def test_export_sodacl(check_expected: str):
data_contract_specification = resolve.resolve_data_contract_from_location(
"./fixtures/sodacl/datacontract.yaml", include_quality=True
"./fixtures/sodacl/datacontract.yaml", inline_quality=True
)

result = to_sodacl_yaml(data_contract_specification)
Expand Down

0 comments on commit 758d974

Please sign in to comment.