Skip to content

Commit

Permalink
Bugfix of importing avro schema that includes an array inside a neste…
Browse files Browse the repository at this point in the history 
…d record or object (datacontract#176)

* added docker-compose.yml and documentation

* add conda environment

* Adding support for debugging with vscode

* bugfix avro import when array type is inside a nested field of the avro schema

* revert conda and vscode debug configurations
  • Loading branch information
@enriquecatala
enriquecatala authored May 6, 2024
1 parent 232997f commit 0c03d0d
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 1 deletion.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -975,4 +975,4 @@ Created by [Stefan Negele](https://www.linkedin.com/in/stefan-negele-573153112/)



<a href="https://github.com/datacontract/datacontract-cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
<a href="https://github.com/datacontract/datacontract-cli" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
2 changes: 2 additions & 0 deletions datacontract/imports/avro_importer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ def map_type_from_avro(avro_type_str: str):
return "boolean"
elif avro_type_str == "record":
return "record"
elif avro_type_str == "array":
return "array"
else:
raise DataContractException(
type="schema",
Expand Down
121 changes: 121 additions & 0 deletions tests/fixtures/avro/data/nested_with_arrays.avsc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
{
"fields": [
{
"name": "Entries",
"type": {
"items": {
"fields": [
{
"name": "Identifier",
"type": {
"logicalType": "uuid",
"type": "string"
}
},
{
"default": null,
"name": "BranchPromo",
"type": [
"null",
{
"fields": [
{
"name": "CodePrefix",
"type": "int"
},
{
"name": "Criteria",
"type": {
"fields": [
{
"default": null,
"name": "MinimumSpendThreshold",
"type": [
"null",
"double"
]
},
{
"default": null,
"name": "ApplicableBranchIDs",
"type": [
"null",
{
"items": "string",
"type": "array"
}
]
},
{
"default": null,
"name": "ProductGroupDetails",
"type": [
"null",
{
"fields": [
{
"name": "IncludesAlcohol",
"type": "boolean"
},
{
"default": null,
"name": "ItemList",
"type": [
"null",
{
"items": {
"fields": [
{
"name": "ProductID",
"type": "string"
},
{
"default": null,
"name": "IsPromoItem",
"type": [
"null",
"boolean"
]
}
],
"name": "ItemRecord",
"namespace": "domain.DemoNamespace.DemoEvent.DemoRecord.CriteriaRecord.ProductGroupDetailsRecord",
"type": "record"
},
"type": "array"
}
]
}
],
"name": "ProductGroupDetailsRecord",
"namespace": "domain.DemoNamespace.DemoEvent.DemoRecord.CriteriaRecord",
"type": "record"
}
]
}
],
"name": "CriteriaRecord",
"namespace": "domain.DemoNamespace.DemoEvent.DemoRecord",
"type": "record"
}
}
],
"name": "DemoRecord",
"namespace": "domain.DemoNamespace.DemoEvent",
"type": "record"
}
]
}
],
"name": "DemoEvent",
"namespace": "domain.DemoNamespace",
"type": "record"
},
"type": "array"
}
}
],
"name": "MarketingLoyaltyAggregation",
"namespace": "domain.schemas",
"type": "record"
}
55 changes: 55 additions & 0 deletions tests/test_import_avro.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,58 @@ def test_import_avro_nested_records():
print("Result:\n", result.to_yaml())
assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected)
assert DataContract(data_contract_str=expected).lint(enabled_linters="none").has_passed()


def test_import_avro_nested_records_with_arrays():
result = DataContract().import_from_source("avro", "fixtures/avro/data/nested_with_arrays.avsc")

expected = """
dataContractSpecification: 0.9.3
id: my-data-contract-id
info:
title: My Data Contract
version: 0.0.1
models:
MarketingLoyaltyAggregation:
namespace: domain.schemas
fields:
Entries:
type: array
required: true
items:
type: object
fields:
Identifier:
type: string
required: true
BranchPromo:
type: record
required: false
fields:
CodePrefix:
type: int
required: true
Criteria:
type: object
required: true
fields:
MinimumSpendThreshold:
type: double
required: false
ApplicableBranchIDs:
type: array
required: false
ProductGroupDetails:
type: record
required: false
fields:
IncludesAlcohol:
type: boolean
required: true
ItemList:
type: array
required: false
"""
print("Result:\n", result.to_yaml())
assert yaml.safe_load(result.to_yaml()) == yaml.safe_load(expected)
assert DataContract(data_contract_str=expected).lint(enabled_linters="none").has_passed()

0 comments on commit 0c03d0d

Please sign in to comment.