From be321ae26790f006ae622280707a664ee3688fb5 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:09:03 -0500 Subject: [PATCH 1/4] build: Upgrade ocdsextensionregistry. Remove milestone documents extension (changing OCDS validations is prohibited). --- common-requirements.txt | 2 +- docs/conf.py | 1 - docs/history/changelog.md | 2 +- .../es/LC_MESSAGES/history/changelog.po | 6 +- .../fr/LC_MESSAGES/history/changelog.po | 3 +- docs/schema/reference.md | 4 - manage.py | 314 +----------------- tests/test_schema_integrity.py | 13 +- 8 files changed, 19 insertions(+), 326 deletions(-) diff --git a/common-requirements.txt b/common-requirements.txt index 3076effd9..c81779f3c 100644 --- a/common-requirements.txt +++ b/common-requirements.txt @@ -90,7 +90,7 @@ myst-parser==0.18.1 # via -r common-requirements.in ocds-babel==0.3.6 # via -r common-requirements.in -ocdsextensionregistry==0.5.0 +ocdsextensionregistry==0.6.0 # via -r common-requirements.in ocdsindex==0.2.0 # via -r common-requirements.in diff --git a/docs/conf.py b/docs/conf.py index c4daa25d7..98d1c159b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -114,7 +114,6 @@ "enquiries": default_extension_version, "location": default_extension_version, "lots": default_extension_version, - "milestone_documents": default_extension_version, "participation_fee": default_extension_version, "process_title": default_extension_version, } diff --git a/docs/history/changelog.md b/docs/history/changelog.md index a717f9dc2..d97ea9573 100644 --- a/docs/history/changelog.md +++ b/docs/history/changelog.md @@ -402,7 +402,7 @@ See the changelogs for: * [Enquiries](https://extensions.open-contracting.org/en/extensions/enquiries/master/#changelog) * [Location](https://extensions.open-contracting.org/en/extensions/location/master/#changelog) * [Lots](https://extensions.open-contracting.org/en/extensions/lots/master/#changelog) -* [Milestone documents](https://extensions.open-contracting.org/en/extensions/milestone_documents/master/#changelog) +* [Milestone documents](https://github.com/open-contracting-extensions/ocds_milestone_documents_extension#changelog) * [Participation fees](https://extensions.open-contracting.org/en/extensions/participation_fee/master/#changelog) * [Process level title and description](https://extensions.open-contracting.org/en/extensions/process_title/master/#changelog) diff --git a/docs/locale/es/LC_MESSAGES/history/changelog.po b/docs/locale/es/LC_MESSAGES/history/changelog.po index e99fd7d4b..7a2399215 100644 --- a/docs/locale/es/LC_MESSAGES/history/changelog.po +++ b/docs/locale/es/LC_MESSAGES/history/changelog.po @@ -792,11 +792,9 @@ msgstr "" #: ../../docs/history/changelog.md:111 msgid "" -"[Milestone documents](https://extensions.open-" -"contracting.org/en/extensions/milestone_documents/master/#changelog)" +"[Milestone documents](https://github.com/open-contracting-extensions/ocds_milestone_documents_extension#changelog)" msgstr "" -"[Documentos de hito](https://extensions.open-" -"contracting.org/en/extensions/milestone_documents/master/#changelog)" +"[Documentos de hito](https://github.com/open-contracting-extensions/ocds_milestone_documents_extension#changelog)" #: ../../docs/history/changelog.md:112 ../../docs/history/changelog.md:190 msgid "" diff --git a/docs/locale/fr/LC_MESSAGES/history/changelog.po b/docs/locale/fr/LC_MESSAGES/history/changelog.po index 9863d3c23..2eb30a8d7 100644 --- a/docs/locale/fr/LC_MESSAGES/history/changelog.po +++ b/docs/locale/fr/LC_MESSAGES/history/changelog.po @@ -601,8 +601,7 @@ msgstr "" #: ../../docs/history/changelog.md:110 msgid "" -"[Milestone documents](https://extensions.open-" -"contracting.org/en/extensions/milestone_documents/master/#changelog)" +"[Milestone documents](https://github.com/open-contracting-extensions/ocds_milestone_documents_extension#changelog)" msgstr "" #: ../../docs/history/changelog.md:111 ../../docs/history/changelog.md:189 diff --git a/docs/schema/reference.md b/docs/schema/reference.md index 698b4754c..e53f6b44a 100644 --- a/docs/schema/reference.md +++ b/docs/schema/reference.md @@ -629,10 +629,6 @@ For delivery milestones, if there is a time frame for delivery, use `.dueAfterDa :collapse: documents ``` -```{extensionlist} The following extensions to milestone are available -:list: milestones -``` - ```{workedexamplelist} The following worked examples are available for milestones :tag: milestone ``` diff --git a/manage.py b/manage.py index 9969859b9..b08b23b3d 100755 --- a/manage.py +++ b/manage.py @@ -9,7 +9,6 @@ import warnings from collections import defaultdict from contextlib import contextmanager -from copy import deepcopy from glob import glob from io import StringIO from pathlib import Path @@ -23,6 +22,7 @@ from babel.messages.pofile import read_po from docutils.utils import relative_path from lxml import etree +from ocdsextensionregistry import get_versioned_release_schema from ocdskit.schema import get_schema_fields basedir = Path(__file__).resolve().parent @@ -40,77 +40,6 @@ def custom_warning_formatter(message, category, filename, lineno, line=None): warnings.formatwarning = custom_warning_formatter -versioned_template = json.loads(""" -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "releaseDate": { - "format": "date-time", - "type": "string" - }, - "releaseID": { - "type": "string" - }, - "value": {}, - "releaseTag": { - "type": "array", - "items": { - "type": "string" - } - } - } - } -} -""") - -common_versioned_definitions = { - "StringNullUriVersioned": { - "type": ["string", "null"], - "format": "uri", - }, - "StringNullDateTimeVersioned": { - "type": ["string", "null"], - "format": "date-time", - }, - "StringNullVersioned": { - "type": ["string", "null"], - "format": None, - }, -} - -recognized_types = ( - # Array - ["array"], - ["array", "null"], # optional string arrays - # Object - ["object"], - ["object", "null"], # /Organization/details - # String - ["string"], - ["string", "null"], - # Literal - ["boolean", "null"], - ["integer", "null"], - ["number", "null"], - # Mixed - ["string", "integer"], - ["string", "integer", "null"], -) - -keywords_to_remove = ( - # Metadata keywords - # https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-6 - "title", - "description", - "default", - # Extended keywords - # http://os4d.opendataservices.coop/development/schema/#extended-json-schema - "omitWhenMerged", - "wholeListMerge", -) - def json_load(filename, library=json, **kwargs): """Load JSON data from the given filename.""" @@ -149,14 +78,6 @@ def get(url): return response -def coerce_to_list(data, key): - """Return the value of the ``key`` key in the ``data`` mapping. If the value is a string, wrap it in an array.""" - item = data.get(key, []) - if isinstance(item, str): - return [item] - return item - - def get_metaschema(): """Patches and returns the JSON Schema Draft 4 metaschema.""" return json_merge_patch.merge( @@ -164,234 +85,6 @@ def get_metaschema(): ) -def get_common_definition_ref(item): - """ - Return a schema that references the common definition that the ``item`` matches: "StringNullUriVersioned", - "StringNullDateTimeVersioned" or "StringNullVersioned". - """ - for name, keywords in common_versioned_definitions.items(): - # If the item matches the definition. - if any(item.get(keyword) != value for keyword, value in keywords.items()): - continue - # And adds no keywords to the definition. - if any(keyword not in {*keywords, *keywords_to_remove} for keyword in item): - continue - return {"$ref": f"#/definitions/{name}"} - return None - - -def add_versioned(schema, unversioned_pointers, pointer=""): - """Call ``_add_versioned`` on each field.""" - for key, value in schema["properties"].items(): - new_pointer = f"{pointer}/properties/{key}" - _add_versioned(schema, unversioned_pointers, new_pointer, key, value) - - for key, value in schema.get("definitions", {}).items(): - new_pointer = f"{pointer}/definitions/{key}" - add_versioned(value, unversioned_pointers, pointer=new_pointer) - - -def _add_versioned(schema, unversioned_pointers, pointer, key, value): - """ - Perform the changes to the schema to refer to versioned/unversioned definitions. - - :param schema dict: the schema of the object on which the field is defined - :param unversioned_pointers set: JSON Pointers to ``id`` fields to leave unversioned if the object is in an array - :param pointer str: the field's pointer - :param key str: the field's name - :param value str: the field's schema - """ - # Skip unversioned fields. - if pointer in unversioned_pointers: - return - - types = coerce_to_list(value, "type") - - # If a type is unrecognized, we might need to update this script. - if ( - "$ref" not in value - and types not in recognized_types - and not (pointer == "/definitions/Quantity/properties/value" and types == ["string", "number", "null"]) - ): - warnings.warn(f"{pointer} has unrecognized type {types}") - - # For example, if $ref is used. - if not types: - # Ignore the `amendment` field, which had no `id` field in OCDS 1.0. - if "deprecated" not in value: - versioned_pointer = f"{value['$ref'][1:]}/properties/id" - # If the `id` field is on an object not in an array, it needs to be versioned (e.g. buyer/properties/id). - if versioned_pointer in unversioned_pointers: - value["$ref"] = value["$ref"] + "VersionedId" - return - - # Reference a common versioned definition if possible, to limit the size of the schema. - ref = get_common_definition_ref(value) - if ref: - schema["properties"][key] = ref - - # Iterate into objects with properties like `Item.unit`. Otherwise, version objects with no properties as a - # whole, like `Organization.details`. - elif types == ["object"] and "properties" in value: - add_versioned(value, unversioned_pointers, pointer=pointer) - - else: - new_value = deepcopy(value) - - if types == ["array"]: - item_types = coerce_to_list(value["items"], "type") - - # See https://standard.open-contracting.org/latest/en/schema/merging/#whole-list-merge - if value.get("wholeListMerge"): - # Update `$ref` to the unversioned definition. - if "$ref" in value["items"]: - new_value["items"]["$ref"] = value["items"]["$ref"] + "Unversioned" - # Otherwise, similarly, don't iterate over item properties. - # See https://standard.open-contracting.org/latest/en/schema/merging/#lists - elif "$ref" in value["items"]: - # Leave `$ref` to the versioned definition. - return - # Exceptional case for deprecated `Amendment.changes`. - elif item_types == ["object"] and pointer == "/definitions/Amendment/properties/changes": - return - # Warn in case new combinations are added to the release schema. - elif item_types != ["string"]: - # Note: Versioning the properties of un-$ref'erenced objects in arrays isn't implemented. However, - # this combination hasn't occurred, with the exception of `Amendment/changes`. - warnings.warn(f"{pointer}/items has unexpected type {item_types}") - - versioned = deepcopy(versioned_template) - versioned["items"]["properties"]["value"] = new_value - schema["properties"][key] = versioned - - -def update_refs_to_unversioned_definitions(schema): - """Replace ``$ref`` values with unversioned definitions.""" - for key, value in schema.items(): - if key == "$ref": - schema[key] = value + "Unversioned" - elif isinstance(value, dict): - update_refs_to_unversioned_definitions(value) - - -def get_unversioned_pointers(schema, fields, pointer=""): - """Return the JSON Pointers to ``id`` fields that must not be versioned if the object is in an array.""" - if isinstance(schema, list): - for index, item in enumerate(schema): - get_unversioned_pointers(item, fields, pointer=f"{pointer}/{index}") - elif isinstance(schema, dict): - # Follows the logic of _get_merge_rules in merge.py from ocds-merge. - types = coerce_to_list(schema, "type") - - # If an array is whole list merge, its items are unversioned. - if "array" in types and schema.get("wholeListMerge"): - return - if "array" in types and "items" in schema: - item_types = coerce_to_list(schema["items"], "type") - # If an array mixes objects and non-objects, it is whole list merge. - if any(item_type != "object" for item_type in item_types): - return - # If it is an array of objects, any `id` fields are unversioned. - if "id" in schema["items"]["properties"]: - if hasattr(schema["items"], "__reference__"): - reference = schema["items"].__reference__["$ref"][1:] - else: - reference = pointer - fields.add(f"{reference}/properties/id") - - for key, value in schema.items(): - get_unversioned_pointers(value, fields, pointer=f"{pointer}/{key}") - - -def remove_omit_when_merged(schema): - """Remove properties that set ``omitWhenMerged``.""" - if isinstance(schema, list): - for item in schema: - remove_omit_when_merged(item) - elif isinstance(schema, dict): - for key, value in schema.items(): - if key == "properties": - for prop in list(value): - if value[prop].get("omitWhenMerged"): - del value[prop] - if prop in schema["required"]: - schema["required"].remove(prop) - remove_omit_when_merged(value) - - -def remove_metadata_and_extended_keywords(schema): - """Remove metadata and extended keywords from properties and definitions.""" - if isinstance(schema, list): - for item in schema: - remove_metadata_and_extended_keywords(item) - elif isinstance(schema, dict): - for key, value in schema.items(): - if key in {"definitions", "properties"}: - for subschema in value.values(): - for keyword in keywords_to_remove: - subschema.pop(keyword, None) - remove_metadata_and_extended_keywords(value) - - -def get_versioned_release_schema(schema): - """Return the versioned release schema.""" - # Update schema metadata. - release_with_underscores = release.replace(".", "__") - schema["id"] = ( - f"https://standard.open-contracting.org/schema/{release_with_underscores}/versioned-release-validation-schema.json" - ) - schema["title"] = "Schema for a compiled, versioned Open Contracting Release." - - # Release IDs, dates and tags appear alongside values in the versioned release schema. - remove_omit_when_merged(schema) - - # Create unversioned copies of all definitions. - unversioned_definitions = {k + "Unversioned": deepcopy(v) for k, v in schema["definitions"].items()} - update_refs_to_unversioned_definitions(unversioned_definitions) - - # Determine which `id` fields occur on objects in arrays. - unversioned_pointers = set() - get_unversioned_pointers(jsonref.replace_refs(schema), unversioned_pointers) - - # Omit `ocid` from versioning. - ocid = schema["properties"].pop("ocid") - add_versioned(schema, unversioned_pointers) - schema["properties"]["ocid"] = ocid - - # Add the common versioned definitions. - for name, keywords in common_versioned_definitions.items(): - versioned = deepcopy(versioned_template) - for keyword, value in keywords.items(): - if value: - versioned["items"]["properties"]["value"][keyword] = value - schema["definitions"][name] = versioned - - # Add missing definitions. - while True: - try: - jsonref.replace_refs(schema, lazy_load=False) - break - except jsonref.JsonRefError as e: - name = e.cause.args[0] - - if name.endswith("VersionedId"): - # Add a copy of an definition with a versioned `id` field, using the same logic as before. - definition = deepcopy(schema["definitions"][name[:-11]]) - pointer = f"/definitions/{name[:-11]}/properties/id" - pointers = unversioned_pointers - {pointer} - _add_versioned(definition, pointers, pointer, "id", definition["properties"]["id"]) - else: - # Add a copy of an definition with no versioned fields. - definition = unversioned_definitions[name] - - schema["definitions"][name] = definition - - # Remove all metadata and extended keywords. - remove_metadata_and_extended_keywords(schema) - - return schema - - @click.group() def cli(): pass @@ -566,7 +259,10 @@ def pre_commit(): json_dump("meta-schema.json", get_metaschema()) json_dump("dereferenced-release-schema.json", jsonref_release_schema) - json_dump("versioned-release-validation-schema.json", get_versioned_release_schema(release_schema)) + json_dump( + "versioned-release-validation-schema.json", + get_versioned_release_schema(release_schema, release.replace(".", "__")), + ) @cli.command() diff --git a/tests/test_schema_integrity.py b/tests/test_schema_integrity.py index c6fdb0f86..789a27688 100644 --- a/tests/test_schema_integrity.py +++ b/tests/test_schema_integrity.py @@ -3,14 +3,19 @@ """ import json -import os.path import sys +from pathlib import Path import jsonref +from ocdsextensionregistry import get_versioned_release_schema -sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +basedir = Path(__file__).resolve().parent.parent -from manage import get_metaschema, get_versioned_release_schema +sys.path.extend([str(basedir), str(basedir / "docs")]) + +from conf import release + +from manage import get_metaschema def test_versioned_release_schema_is_in_sync(): @@ -18,7 +23,7 @@ def test_versioned_release_schema_is_in_sync(): actual = json.load(f) with open("schema/release-schema.json") as f: - expected = get_versioned_release_schema(json.load(f)) + expected = get_versioned_release_schema(json.load(f), release.replace(".", "__")) assert actual == expected, "Run: python manage.py pre-commit" From 39a7163f781ca15e8d555c78365c72ac9a976131 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:12:24 -0500 Subject: [PATCH 2/4] build: Upgrade sphinxcontrib-opendataservices-jsonschema --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9c2ea5690..1c7a3ab40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ ocdskit==1.1.3 sphinx-design==0.4.1 sphinxcontrib-opencontracting==0.0.8 -sphinxcontrib-opendataservices-jsonschema==0.6.1 +sphinxcontrib-opendataservices-jsonschema==0.7.1 sphinxcontrib-opendataservices==0.5.0 From 311c9d2fa22f15ce68c964704e4959170a23876d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:14:20 -0500 Subject: [PATCH 3/4] chore: Add noqa for local imports --- tests/test_schema_integrity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_schema_integrity.py b/tests/test_schema_integrity.py index 789a27688..fdbab6b76 100644 --- a/tests/test_schema_integrity.py +++ b/tests/test_schema_integrity.py @@ -13,9 +13,9 @@ sys.path.extend([str(basedir), str(basedir / "docs")]) -from conf import release +from conf import release # noqa: E402 -from manage import get_metaschema +from manage import get_metaschema # noqa: E402 def test_versioned_release_schema_is_in_sync(): From 561a0520508107c3d230ec4c5c206559d281be4d Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:19:25 -0500 Subject: [PATCH 4/4] build: Upgrade ocdsextensionregistry --- common-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common-requirements.txt b/common-requirements.txt index c81779f3c..fb836f0ad 100644 --- a/common-requirements.txt +++ b/common-requirements.txt @@ -90,7 +90,7 @@ myst-parser==0.18.1 # via -r common-requirements.in ocds-babel==0.3.6 # via -r common-requirements.in -ocdsextensionregistry==0.6.0 +ocdsextensionregistry==0.6.1 # via -r common-requirements.in ocdsindex==0.2.0 # via -r common-requirements.in @@ -158,7 +158,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -standard-theme @ git+https://github.com/open-contracting/standard_theme.git@c3cbd1b8ba6db24624e5d838ed18147db63f5d1b#egg=standard_theme +standard-theme @ git+https://github.com/open-contracting/standard_theme.git@5de343d1d8e342b5f2a42c6132db37aebe382e36#egg=standard_theme # via -r common-requirements.in starlette==0.40.0 # via sphinx-autobuild