From 96bf6e6fb0b0fbf09bdd2d978fd8f92bc6bb5a6e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 16 May 2024 14:56:50 +0200 Subject: [PATCH] First support for server-specific types in a config map. Resolves #150 --- datacontract/export/sql_converter.py | 1 + datacontract/export/sql_type_converter.py | 5 +- .../model/data_contract_specification.py | 3 +- tests/fixtures/snowflake/datacontract.yaml | 6 ++ tests/test_download_datacontract_file.py | 56 +++++++------------ tests/test_export_sql.py | 3 +- tests/test_export_sql_query.py | 7 ++- 7 files changed, 40 insertions(+), 41 deletions(-) diff --git a/datacontract/export/sql_converter.py b/datacontract/export/sql_converter.py index 4cd6b19c..0eb9190a 100644 --- a/datacontract/export/sql_converter.py +++ b/datacontract/export/sql_converter.py @@ -63,6 +63,7 @@ def to_sql_ddl(data_contract_spec: DataContractSpecification, server_type: str = result = "" result += f"-- Data Contract: {data_contract_spec.id}\n" result += f"-- SQL Dialect: {server_type}\n" + for model_name, model in iter(data_contract_spec.models.items()): result += _to_sql_table(table_prefix + model_name, model, server_type) diff --git a/datacontract/export/sql_type_converter.py b/datacontract/export/sql_type_converter.py index b7326298..161f2b56 100644 --- a/datacontract/export/sql_type_converter.py +++ b/datacontract/export/sql_type_converter.py @@ -15,7 +15,10 @@ def convert_to_sql_type(field: Field, server_type: str) -> str: # snowflake data types: # https://docs.snowflake.com/en/sql-reference/data-types.html -def convert_to_snowflake(field) -> None | str: +def convert_to_snowflake(field: Field) -> None | str: + if field.config and field.config["snowflakeType"] is not None: + return field.config["snowflakeType"] + type = field.type # currently optimized for snowflake # LEARNING: data contract has no direct support for CHAR,CHARACTER diff --git a/datacontract/model/data_contract_specification.py b/datacontract/model/data_contract_specification.py index bc79ade6..32ca400b 100644 --- a/datacontract/model/data_contract_specification.py +++ b/datacontract/model/data_contract_specification.py @@ -1,5 +1,5 @@ import os -from typing import List, Dict, Optional +from typing import List, Dict, Optional, Any import pydantic as pyd import yaml @@ -86,6 +86,7 @@ class Field(pyd.BaseModel): items: "Field" = None precision: int = None scale: int = None + config: Dict[str, Any] = None class Model(pyd.BaseModel): diff --git a/tests/fixtures/snowflake/datacontract.yaml b/tests/fixtures/snowflake/datacontract.yaml index 4a389246..33273b29 100644 --- a/tests/fixtures/snowflake/datacontract.yaml +++ b/tests/fixtures/snowflake/datacontract.yaml @@ -45,6 +45,12 @@ models: type: text format: email required: true + PROCESSING_TIMESTAMP: + description: The processing timestamp in the current session’s time zone. + type: timestamp + required: true + config: + snowflakeType: TIMESTAMP_LTZ line_items: description: A single article that is part of an order. type: table diff --git a/tests/test_download_datacontract_file.py b/tests/test_download_datacontract_file.py index 5dfb5034..72f517b6 100644 --- a/tests/test_download_datacontract_file.py +++ b/tests/test_download_datacontract_file.py @@ -1,61 +1,47 @@ -import os - import requests from typer.testing import CliRunner from datacontract.cli import app runner = CliRunner() -_datacontract_test_path = "./.tmp/datacontract.yaml" _default_template_url = "https://datacontract.com/datacontract.init.yaml" _custom_template_url = "https://studio.datacontract.com/s/ef47b7ea-879c-48d5-adf4-aa68b000b00f.yaml" -def test_download_datacontract_file_with_defaults(): - _setup() - - runner.invoke(app, ["init", _datacontract_test_path]) - - _compare_test_datacontract_with(_default_template_url) - - -def test_download_datacontract_file_from_custom_url(): - _setup() +def test_download_datacontract_file_with_defaults(tmp_path): + datacontract_test_path = tmp_path / "datacontract.yaml" + runner.invoke(app, ["init", str(datacontract_test_path)]) + _compare_test_datacontract_with(str(datacontract_test_path), _default_template_url) - runner.invoke(app, ["init", _datacontract_test_path, "--template", _custom_template_url]) - _compare_test_datacontract_with(_custom_template_url) +def test_download_datacontract_file_from_custom_url(tmp_path): + datacontract_test_path = tmp_path / "datacontract.yaml" + runner.invoke(app, ["init", str(datacontract_test_path), "--template", _custom_template_url]) + _compare_test_datacontract_with(str(datacontract_test_path), _custom_template_url) -def test_download_datacontract_file_file_exists(): - _setup() - +def test_download_datacontract_file_file_exists(tmp_path): + datacontract_test_path = tmp_path / "datacontract.yaml" # invoke twice to produce error - runner.invoke(app, ["init", _datacontract_test_path]) - result = runner.invoke(app, ["init", _datacontract_test_path, "--template", _custom_template_url]) + runner.invoke(app, ["init", str(datacontract_test_path)]) + result = runner.invoke(app, ["init", str(datacontract_test_path), "--template", _custom_template_url]) assert result.exit_code == 1 assert "File already exists, use --overwrite to overwrite" in result.stdout - _compare_test_datacontract_with(_default_template_url) - + _compare_test_datacontract_with(str(datacontract_test_path), _default_template_url) -def test_download_datacontract_file_overwrite_file(): - _setup() - runner.invoke(app, ["init", _datacontract_test_path]) - result = runner.invoke(app, ["init", _datacontract_test_path, "--template", _custom_template_url, "--overwrite"]) +def test_download_datacontract_file_overwrite_file(tmp_path): + datacontract_test_path = tmp_path / "datacontract.yaml" + runner.invoke(app, ["init", str(datacontract_test_path)]) + result = runner.invoke(app, + ["init", str(datacontract_test_path), "--template", _custom_template_url, "--overwrite"]) assert result.exit_code == 0 - _compare_test_datacontract_with(_custom_template_url) - - -def _setup(): - os.makedirs(".tmp", exist_ok=True) - if os.path.exists(_datacontract_test_path): - os.remove(_datacontract_test_path) + _compare_test_datacontract_with(str(datacontract_test_path), _custom_template_url) -def _compare_test_datacontract_with(url: str): +def _compare_test_datacontract_with(datacontract_test_path, url: str): text = requests.get(url).text - with open(_datacontract_test_path, "r") as tmp: + with open(datacontract_test_path, "r") as tmp: assert tmp.read().replace("\r", "") == text.replace("\r", "") diff --git a/tests/test_export_sql.py b/tests/test_export_sql.py index c77b788e..1e1e46de 100644 --- a/tests/test_export_sql.py +++ b/tests/test_export_sql.py @@ -38,7 +38,8 @@ def test_to_sql_ddl_snowflake(): ORDER_TIMESTAMP TIMESTAMP_TZ not null, ORDER_TOTAL NUMBER not null, CUSTOMER_ID TEXT, - CUSTOMER_EMAIL_ADDRESS TEXT not null + CUSTOMER_EMAIL_ADDRESS TEXT not null, + PROCESSING_TIMESTAMP TIMESTAMP_LTZ not null ); CREATE TABLE line_items ( LINE_ITEM_ID TEXT not null, diff --git a/tests/test_export_sql_query.py b/tests/test_export_sql_query.py index 3d4e6181..1459744d 100644 --- a/tests/test_export_sql_query.py +++ b/tests/test_export_sql_query.py @@ -14,7 +14,7 @@ def test_cli(): assert result.exit_code == 0 -def test_to_sql_ddl_postgres(): +def test_to_sql_query_postgres(): actual = DataContract(data_contract_file="fixtures/postgres-export/datacontract.yaml").export("sql-query") expected = """ -- Data Contract: postgres @@ -28,7 +28,7 @@ def test_to_sql_ddl_postgres(): assert actual.strip() == expected.strip() -def test_to_sql_ddl_snowflake(): +def test_to_sql_query_snowflake(): actual = DataContract(data_contract_file="fixtures/snowflake/datacontract.yaml").export("sql-query", model="orders") expected = """ -- Data Contract: urn:datacontract:checkout:snowflake_orders_pii_v2 @@ -38,7 +38,8 @@ def test_to_sql_ddl_snowflake(): ORDER_TIMESTAMP, ORDER_TOTAL, CUSTOMER_ID, - CUSTOMER_EMAIL_ADDRESS + CUSTOMER_EMAIL_ADDRESS, + PROCESSING_TIMESTAMP from orders """ assert actual.strip() == expected.strip()