From 9faaa1db0e8f4e5f68edc1ad5e78c6f5a8f70735 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Nov 2022 17:41:52 +0000 Subject: [PATCH] jsontogeojson: Add meta output, with output field coverage Helpful for Open-Telecoms-Data/cove-ofds#6 --- CHANGELOG.md | 2 + libcoveofds/cli.py | 9 + libcoveofds/geojson.py | 20 ++ make_expected_test_data.sh | 8 +- .../basic_1.expected.meta.json | 120 +++++++++++ .../no_geometry_1.expected.meta.json | 90 ++++++++ .../organisations_1.expected.meta.json | 180 ++++++++++++++++ .../phases_1.expected.meta.json | 204 ++++++++++++++++++ tests/test_json_to_geojson.py | 10 + 9 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/json_to_geojson/basic_1.expected.meta.json create mode 100644 tests/fixtures/json_to_geojson/no_geometry_1.expected.meta.json create mode 100644 tests/fixtures/json_to_geojson/organisations_1.expected.meta.json create mode 100644 tests/fixtures/json_to_geojson/phases_1.expected.meta.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 53b6456..0a852f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add meta output, with output field coverage - Fix bug that meant get_json() could not be called twice - Include contracts +- Improved in JSON to GeoJSON: + - Add meta output, with output field coverage ### Changed diff --git a/libcoveofds/cli.py b/libcoveofds/cli.py index fd6fcb7..14ea671 100644 --- a/libcoveofds/cli.py +++ b/libcoveofds/cli.py @@ -39,6 +39,11 @@ def main(): json_to_geojson_parser.add_argument( "outputspansfilename", help="Output filename to write Spans GeoJSON data to" ) + json_to_geojson_parser.add_argument( + "--outputmetafilename", + help="Output filename to write meta JSON data to", + required=False, + ) geojson_to_json_parser = subparsers.add_parser("geojsontojson", aliases=["gjtoj"]) geojson_to_json_parser.add_argument( @@ -110,6 +115,10 @@ def main(): with open(args.outputspansfilename, "w") as fp: json.dump(converter.get_spans_geojson(), fp, indent=4) + if args.outputmetafilename: + with open(args.outputmetafilename, "w") as fp: + json.dump(converter.get_meta_json(), fp, indent=4) + elif args.subparser_name == "geojsontojson" or args.subparser_name == "gjtoj": with open(args.inputnodesfilename) as fp: diff --git a/libcoveofds/geojson.py b/libcoveofds/geojson.py index cb5ad26..96d812d 100644 --- a/libcoveofds/geojson.py +++ b/libcoveofds/geojson.py @@ -49,6 +49,26 @@ def get_nodes_geojson(self) -> dict: def get_spans_geojson(self) -> dict: return {"type": "FeatureCollection", "features": self._spans_geojson_features} + def get_meta_json(self) -> dict: + out: dict = { + "nodes_output_field_coverage": {}, + "spans_output_field_coverage": {}, + } + # nodes field coverage + for key, value in fields_present_generator(self.get_nodes_geojson()): + if key not in out["nodes_output_field_coverage"]: + out["nodes_output_field_coverage"][key] = {"count": 1} + else: + out["nodes_output_field_coverage"][key]["count"] += 1 + # spans field coverage + for key, value in fields_present_generator(self.get_spans_geojson()): + if key not in out["spans_output_field_coverage"]: + out["spans_output_field_coverage"][key] = {"count": 1} + else: + out["spans_output_field_coverage"][key]["count"] += 1 + # return + return out + def _dereference_object(self, ref, list): """ Return from list the object referenced by ref. Otherwise, return ref. diff --git a/make_expected_test_data.sh b/make_expected_test_data.sh index 568a202..04da812 100755 --- a/make_expected_test_data.sh +++ b/make_expected_test_data.sh @@ -18,10 +18,10 @@ libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/phases_1.m libcoveofds gjtoj --outputmetafilename tests/fixtures/geojson_to_json/organisations_1.meta.expected.json tests/fixtures/geojson_to_json/organisations_1.nodes.geo.json tests/fixtures/geojson_to_json/organisations_1.spans.geo.json tests/fixtures/geojson_to_json/organisations_1.expected.json # JSON to GeoJSON -libcoveofds jtogj tests/fixtures/json_to_geojson/basic_1.json tests/fixtures/json_to_geojson/basic_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/basic_1.expected.spans.geo.json -libcoveofds jtogj tests/fixtures/json_to_geojson/phases_1.json tests/fixtures/json_to_geojson/phases_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/phases_1.expected.spans.geo.json -libcoveofds jtogj tests/fixtures/json_to_geojson/organisations_1.json tests/fixtures/json_to_geojson/organisations_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/organisations_1.expected.spans.geo.json -libcoveofds jtogj tests/fixtures/json_to_geojson/no_geometry_1.json tests/fixtures/json_to_geojson/no_geometry_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/no_geometry_1.expected.spans.geo.json +libcoveofds jtogj --outputmetafilename tests/fixtures/json_to_geojson/basic_1.expected.meta.json tests/fixtures/json_to_geojson/basic_1.json tests/fixtures/json_to_geojson/basic_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/basic_1.expected.spans.geo.json +libcoveofds jtogj --outputmetafilename tests/fixtures/json_to_geojson/phases_1.expected.meta.json tests/fixtures/json_to_geojson/phases_1.json tests/fixtures/json_to_geojson/phases_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/phases_1.expected.spans.geo.json +libcoveofds jtogj --outputmetafilename tests/fixtures/json_to_geojson/organisations_1.expected.meta.json tests/fixtures/json_to_geojson/organisations_1.json tests/fixtures/json_to_geojson/organisations_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/organisations_1.expected.spans.geo.json +libcoveofds jtogj --outputmetafilename tests/fixtures/json_to_geojson/no_geometry_1.expected.meta.json tests/fixtures/json_to_geojson/no_geometry_1.json tests/fixtures/json_to_geojson/no_geometry_1.expected.nodes.geo.json tests/fixtures/json_to_geojson/no_geometry_1.expected.spans.geo.json # JSON Schema validate libcoveofds jsv tests/fixtures/jsonschemavalidate/basic_1.input.json > tests/fixtures/jsonschemavalidate/basic_1.expected.json diff --git a/tests/fixtures/json_to_geojson/basic_1.expected.meta.json b/tests/fixtures/json_to_geojson/basic_1.expected.meta.json new file mode 100644 index 0000000..27a754b --- /dev/null +++ b/tests/fixtures/json_to_geojson/basic_1.expected.meta.json @@ -0,0 +1,120 @@ +{ + "nodes_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 2 + }, + "/features/geometry": { + "count": 2 + }, + "/features/geometry/type": { + "count": 2 + }, + "/features/geometry/coordinates": { + "count": 2 + }, + "/features/properties": { + "count": 2 + }, + "/features/properties/id": { + "count": 2 + }, + "/features/properties/name": { + "count": 2 + }, + "/features/properties/network": { + "count": 2 + }, + "/features/properties/network/id": { + "count": 2 + }, + "/features/properties/network/name": { + "count": 2 + }, + "/features/properties/status": { + "count": 1 + } + }, + "spans_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 1 + }, + "/features/geometry": { + "count": 1 + }, + "/features/geometry/type": { + "count": 1 + }, + "/features/geometry/coordinates": { + "count": 1 + }, + "/features/properties": { + "count": 1 + }, + "/features/properties/id": { + "count": 1 + }, + "/features/properties/name": { + "count": 1 + }, + "/features/properties/start": { + "count": 1 + }, + "/features/properties/start/id": { + "count": 1 + }, + "/features/properties/start/name": { + "count": 1 + }, + "/features/properties/start/location": { + "count": 1 + }, + "/features/properties/start/location/type": { + "count": 1 + }, + "/features/properties/start/location/coordinates": { + "count": 1 + }, + "/features/properties/end": { + "count": 1 + }, + "/features/properties/end/id": { + "count": 1 + }, + "/features/properties/end/name": { + "count": 1 + }, + "/features/properties/end/status": { + "count": 1 + }, + "/features/properties/end/location": { + "count": 1 + }, + "/features/properties/end/location/type": { + "count": 1 + }, + "/features/properties/end/location/coordinates": { + "count": 1 + }, + "/features/properties/network": { + "count": 1 + }, + "/features/properties/network/id": { + "count": 1 + }, + "/features/properties/network/name": { + "count": 1 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/json_to_geojson/no_geometry_1.expected.meta.json b/tests/fixtures/json_to_geojson/no_geometry_1.expected.meta.json new file mode 100644 index 0000000..018ae8a --- /dev/null +++ b/tests/fixtures/json_to_geojson/no_geometry_1.expected.meta.json @@ -0,0 +1,90 @@ +{ + "nodes_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 2 + }, + "/features/geometry": { + "count": 2 + }, + "/features/properties": { + "count": 2 + }, + "/features/properties/id": { + "count": 2 + }, + "/features/properties/name": { + "count": 2 + }, + "/features/properties/network": { + "count": 2 + }, + "/features/properties/network/id": { + "count": 2 + }, + "/features/properties/network/name": { + "count": 2 + }, + "/features/properties/status": { + "count": 1 + } + }, + "spans_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 1 + }, + "/features/geometry": { + "count": 1 + }, + "/features/properties": { + "count": 1 + }, + "/features/properties/id": { + "count": 1 + }, + "/features/properties/name": { + "count": 1 + }, + "/features/properties/start": { + "count": 1 + }, + "/features/properties/start/id": { + "count": 1 + }, + "/features/properties/start/name": { + "count": 1 + }, + "/features/properties/end": { + "count": 1 + }, + "/features/properties/end/id": { + "count": 1 + }, + "/features/properties/end/name": { + "count": 1 + }, + "/features/properties/end/status": { + "count": 1 + }, + "/features/properties/network": { + "count": 1 + }, + "/features/properties/network/id": { + "count": 1 + }, + "/features/properties/network/name": { + "count": 1 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/json_to_geojson/organisations_1.expected.meta.json b/tests/fixtures/json_to_geojson/organisations_1.expected.meta.json new file mode 100644 index 0000000..22cc0ce --- /dev/null +++ b/tests/fixtures/json_to_geojson/organisations_1.expected.meta.json @@ -0,0 +1,180 @@ +{ + "nodes_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 2 + }, + "/features/geometry": { + "count": 2 + }, + "/features/geometry/type": { + "count": 2 + }, + "/features/geometry/coordinates": { + "count": 2 + }, + "/features/properties": { + "count": 2 + }, + "/features/properties/id": { + "count": 2 + }, + "/features/properties/name": { + "count": 2 + }, + "/features/properties/physicalInfrastructureProvider": { + "count": 2 + }, + "/features/properties/physicalInfrastructureProvider/id": { + "count": 2 + }, + "/features/properties/physicalInfrastructureProvider/name": { + "count": 2 + }, + "/features/properties/networkProvider": { + "count": 2 + }, + "/features/properties/networkProvider/id": { + "count": 2 + }, + "/features/properties/networkProvider/name": { + "count": 2 + }, + "/features/properties/network": { + "count": 2 + }, + "/features/properties/network/id": { + "count": 2 + }, + "/features/properties/network/name": { + "count": 2 + }, + "/features/properties/status": { + "count": 1 + } + }, + "spans_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 1 + }, + "/features/geometry": { + "count": 1 + }, + "/features/geometry/type": { + "count": 1 + }, + "/features/geometry/coordinates": { + "count": 1 + }, + "/features/properties": { + "count": 1 + }, + "/features/properties/id": { + "count": 1 + }, + "/features/properties/name": { + "count": 1 + }, + "/features/properties/start": { + "count": 1 + }, + "/features/properties/start/id": { + "count": 1 + }, + "/features/properties/start/name": { + "count": 1 + }, + "/features/properties/start/location": { + "count": 1 + }, + "/features/properties/start/location/type": { + "count": 1 + }, + "/features/properties/start/location/coordinates": { + "count": 1 + }, + "/features/properties/start/physicalInfrastructureProvider": { + "count": 1 + }, + "/features/properties/start/physicalInfrastructureProvider/id": { + "count": 1 + }, + "/features/properties/start/networkProvider": { + "count": 1 + }, + "/features/properties/start/networkProvider/id": { + "count": 1 + }, + "/features/properties/end": { + "count": 1 + }, + "/features/properties/end/id": { + "count": 1 + }, + "/features/properties/end/name": { + "count": 1 + }, + "/features/properties/end/status": { + "count": 1 + }, + "/features/properties/end/location": { + "count": 1 + }, + "/features/properties/end/location/type": { + "count": 1 + }, + "/features/properties/end/location/coordinates": { + "count": 1 + }, + "/features/properties/end/physicalInfrastructureProvider": { + "count": 1 + }, + "/features/properties/end/physicalInfrastructureProvider/id": { + "count": 1 + }, + "/features/properties/end/networkProvider": { + "count": 1 + }, + "/features/properties/end/networkProvider/id": { + "count": 1 + }, + "/features/properties/physicalInfrastructureProvider": { + "count": 1 + }, + "/features/properties/physicalInfrastructureProvider/id": { + "count": 1 + }, + "/features/properties/physicalInfrastructureProvider/name": { + "count": 1 + }, + "/features/properties/networkProvider": { + "count": 1 + }, + "/features/properties/networkProvider/id": { + "count": 1 + }, + "/features/properties/networkProvider/name": { + "count": 1 + }, + "/features/properties/network": { + "count": 1 + }, + "/features/properties/network/id": { + "count": 1 + }, + "/features/properties/network/name": { + "count": 1 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/json_to_geojson/phases_1.expected.meta.json b/tests/fixtures/json_to_geojson/phases_1.expected.meta.json new file mode 100644 index 0000000..0705023 --- /dev/null +++ b/tests/fixtures/json_to_geojson/phases_1.expected.meta.json @@ -0,0 +1,204 @@ +{ + "nodes_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 2 + }, + "/features/geometry": { + "count": 2 + }, + "/features/geometry/type": { + "count": 2 + }, + "/features/geometry/coordinates": { + "count": 2 + }, + "/features/properties": { + "count": 2 + }, + "/features/properties/id": { + "count": 2 + }, + "/features/properties/name": { + "count": 2 + }, + "/features/properties/phase": { + "count": 2 + }, + "/features/properties/phase/id": { + "count": 2 + }, + "/features/properties/phase/name": { + "count": 2 + }, + "/features/properties/phase/description": { + "count": 2 + }, + "/features/properties/network": { + "count": 2 + }, + "/features/properties/network/id": { + "count": 2 + }, + "/features/properties/network/name": { + "count": 2 + }, + "/features/properties/network/contracts": { + "count": 2 + }, + "/features/properties/network/contracts/id": { + "count": 2 + }, + "/features/properties/network/contracts/title": { + "count": 2 + }, + "/features/properties/network/contracts/description": { + "count": 2 + }, + "/features/properties/network/contracts/relatedPhases": { + "count": 2 + }, + "/features/properties/network/contracts/relatedPhases/id": { + "count": 2 + }, + "/features/properties/network/contracts/relatedPhases/name": { + "count": 2 + }, + "/features/properties/network/contracts/relatedPhases/description": { + "count": 2 + }, + "/features/properties/status": { + "count": 1 + } + }, + "spans_output_field_coverage": { + "/type": { + "count": 1 + }, + "/features": { + "count": 1 + }, + "/features/type": { + "count": 1 + }, + "/features/geometry": { + "count": 1 + }, + "/features/geometry/type": { + "count": 1 + }, + "/features/geometry/coordinates": { + "count": 1 + }, + "/features/properties": { + "count": 1 + }, + "/features/properties/id": { + "count": 1 + }, + "/features/properties/name": { + "count": 1 + }, + "/features/properties/start": { + "count": 1 + }, + "/features/properties/start/id": { + "count": 1 + }, + "/features/properties/start/name": { + "count": 1 + }, + "/features/properties/start/location": { + "count": 1 + }, + "/features/properties/start/location/type": { + "count": 1 + }, + "/features/properties/start/location/coordinates": { + "count": 1 + }, + "/features/properties/start/phase": { + "count": 1 + }, + "/features/properties/start/phase/id": { + "count": 1 + }, + "/features/properties/end": { + "count": 1 + }, + "/features/properties/end/id": { + "count": 1 + }, + "/features/properties/end/name": { + "count": 1 + }, + "/features/properties/end/status": { + "count": 1 + }, + "/features/properties/end/location": { + "count": 1 + }, + "/features/properties/end/location/type": { + "count": 1 + }, + "/features/properties/end/location/coordinates": { + "count": 1 + }, + "/features/properties/end/phase": { + "count": 1 + }, + "/features/properties/end/phase/id": { + "count": 1 + }, + "/features/properties/phase": { + "count": 1 + }, + "/features/properties/phase/id": { + "count": 1 + }, + "/features/properties/phase/name": { + "count": 1 + }, + "/features/properties/phase/description": { + "count": 1 + }, + "/features/properties/network": { + "count": 1 + }, + "/features/properties/network/id": { + "count": 1 + }, + "/features/properties/network/name": { + "count": 1 + }, + "/features/properties/network/contracts": { + "count": 1 + }, + "/features/properties/network/contracts/id": { + "count": 1 + }, + "/features/properties/network/contracts/title": { + "count": 1 + }, + "/features/properties/network/contracts/description": { + "count": 1 + }, + "/features/properties/network/contracts/relatedPhases": { + "count": 1 + }, + "/features/properties/network/contracts/relatedPhases/id": { + "count": 1 + }, + "/features/properties/network/contracts/relatedPhases/name": { + "count": 1 + }, + "/features/properties/network/contracts/relatedPhases/description": { + "count": 1 + } + } +} \ No newline at end of file diff --git a/tests/test_json_to_geojson.py b/tests/test_json_to_geojson.py index 93421da..dc70d8c 100644 --- a/tests/test_json_to_geojson.py +++ b/tests/test_json_to_geojson.py @@ -40,6 +40,12 @@ def test_json_to_geojson(filename): "json_to_geojson", filename + ".expected.nodes.geo.json", ) + meta_expected_filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "fixtures", + "json_to_geojson", + filename + ".expected.meta.json", + ) with open(json_filename) as fp: json_data = json.load(fp) @@ -54,3 +60,7 @@ def test_json_to_geojson(filename): with open(expected_nodes_filename) as fp: expected_nodes_data = json.load(fp) assert expected_nodes_data == converter.get_nodes_geojson() + + with open(meta_expected_filename) as fp: + meta_expected_data = json.load(fp) + assert meta_expected_data == converter.get_meta_json()