Skip to content

Commit

Permalink
Add diagnostics to check result
Browse files Browse the repository at this point in the history
  • Loading branch information
jochenchrist committed Apr 15, 2024
1 parent 5d8e868 commit 2216e3f
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 8 deletions.
20 changes: 14 additions & 6 deletions datacontract/engines/soda/check_soda_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from datacontract.model.run import Run, Check, Log


def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark: SparkSession, tmp_dir):
def check_soda_execute(run: Run, data_contract: DataContractSpecification, server: Server, spark: SparkSession,
tmp_dir):
if data_contract is None:
run.log_warn("Cannot run engine soda-core, as data contract is invalid")
return
Expand Down Expand Up @@ -107,16 +108,13 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
for c in scan_results.get("checks"):
check = Check(
type="schema",
result="passed"
if c.get("outcome") == "pass"
else "failed"
if c.get("outcome") == "fail"
else c.get("outcome"),
result=to_result(c),
reason=", ".join(c.get("outcomeReasons")),
name=c.get("name"),
model=c.get("table"),
field=c.get("column"),
engine="soda-core",
diagnostics=c.get("diagnostics"),
)
update_reason(check, c)
run.checks.append(check)
Expand Down Expand Up @@ -144,6 +142,16 @@ def check_soda_execute(run: Run, data_contract: DataContractSpecification, serve
return


def to_result(c) -> str:
soda_outcome = c.get("outcome")
if soda_outcome == "pass":
return "passed"
elif soda_outcome == "fail":
return "failed"
else:
return soda_outcome


def update_reason(check, c):
"""Try to find a reason in diagnostics"""
if check.result == "passed":
Expand Down
3 changes: 2 additions & 1 deletion datacontract/model/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class Check(BaseModel):
model: Optional[str] = None
field: Optional[str] = None
details: Optional[str] = None
diagnostics: Optional[dict] = None


class Log(BaseModel):
Expand Down Expand Up @@ -69,7 +70,7 @@ def log_error(self, message: str):
self.logs.append(Log(level="ERROR", message=message, timestamp=datetime.now(timezone.utc)))

def pretty(self):
return self.model_dump_json()
return self.model_dump_json(indent=2)

@staticmethod
def create_run():
Expand Down
2 changes: 1 addition & 1 deletion tests/test_test_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def test_valid():
# publish=True,
)
run = data_contract.test()
print(run)
print(run.pretty())
assert run.result == "passed"
assert len(run.checks) == 4
assert all(check.result == "passed" for check in run.checks)

0 comments on commit 2216e3f

Please sign in to comment.