diff --git a/apps_ci/scripts/catalog_update.py b/apps_ci/scripts/catalog_update.py index 3c09d6d..fd8662c 100644 --- a/apps_ci/scripts/catalog_update.py +++ b/apps_ci/scripts/catalog_update.py @@ -26,7 +26,8 @@ def get_trains(location: str) -> typing.Tuple[dict, dict]: catalog_data = {} versions_data = {} for train_name, train_data in retrieve_trains_data( - get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse + get_apps_in_trains(trains_to_traverse, location), location, preferred_trains, trains_to_traverse, + normalize_questions=False, )[0].items(): catalog_data[train_name] = {} versions_data[train_name] = {} diff --git a/catalog_reader/app.py b/catalog_reader/app.py index d9ff930..d23a8ee 100644 --- a/catalog_reader/app.py +++ b/catalog_reader/app.py @@ -19,7 +19,7 @@ def get_app_details( - item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None + item_location: str, questions_context: typing.Optional[dict] = None, options: typing.Optional[dict] = None, ) -> dict: catalog_path = item_location.rstrip('/').rsplit('/', 2)[0] item = item_location.rsplit('/', 1)[-1] @@ -51,6 +51,7 @@ def get_app_details( item_data.update(get_app_details_impl(item_location, schema, questions_context, { 'retrieve_latest_version': not retrieve_versions, 'default_values_callable': options.get('default_values_callable'), + 'normalize_questions': options.get('normalize_questions', True), })) unhealthy_versions = [] desired_keys_mapping = { @@ -134,7 +135,11 @@ def get_app_details_impl( version_details.update({ 'healthy': True, - **get_app_version_details(version_details['location'], questions_context) + **get_app_version_details( + version_details['location'], questions_context, { + 'normalize_questions': options.get('normalize_questions', True), + }, + ) }) if retrieve_latest_version: break @@ -145,6 +150,7 @@ def get_app_details_impl( def get_app_version_details( version_path: str, questions_context: typing.Optional[dict], options: typing.Optional[dict] = None ) -> dict: + options = options or {} version_data = {'location': version_path, 'required_features': set()} for key, filename, parser in ( ('app_metadata', 'app.yaml', yaml.safe_load), @@ -160,13 +166,14 @@ def get_app_version_details( # We will normalise questions now so that if they have any references, we render them accordingly # like a field referring to available interfaces on the system - normalize_questions(version_data, questions_context or get_default_questions_context()) + if options.get('normalize_questions', True): + normalize_questions(version_data, questions_context or get_default_questions_context()) version_data.update({ 'supported': version_supported(version_data), 'required_features': list(version_data['required_features']), }) - if options and options.get('default_values_callable'): + if options.get('default_values_callable'): version_data['values'] = options['default_values_callable'](version_data) app_metadata = version_data['app_metadata'] diff --git a/catalog_reader/catalog.py b/catalog_reader/catalog.py index 631d680..6183978 100644 --- a/catalog_reader/catalog.py +++ b/catalog_reader/catalog.py @@ -8,11 +8,15 @@ from .train_utils import get_train_path, is_train_valid -def app_details(apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str) -> dict: +def app_details( + apps: dict, location: str, questions_context: typing.Optional[dict], app_key: str, normalize_questions: bool = True, +) -> dict: train = apps[app_key] app = app_key.removesuffix(f'_{train}') app_location = os.path.join(get_train_path(location), train, app) - return get_app_details(app_location, questions_context, {'retrieve_versions': True}) + return get_app_details( + app_location, questions_context, {'retrieve_versions': True, 'normalize_questions': normalize_questions}, + ) def retrieve_train_names(location: str, all_trains=True, trains_filter=None) -> list: @@ -37,8 +41,8 @@ def get_apps_in_trains(trains_to_traverse: list, catalog_location: str) -> dict: def retrieve_trains_data( - apps: dict, catalog_location: str, preferred_trains: list, - trains_to_traverse: list, job: typing.Any = None, questions_context: typing.Optional[dict] = None + apps: dict, catalog_location: str, preferred_trains: list, trains_to_traverse: list, job: typing.Any = None, + questions_context: typing.Optional[dict] = None, normalize_questions: bool = True, ) -> typing.Tuple[dict, set]: questions_context = questions_context or get_default_questions_context() trains = { @@ -50,7 +54,9 @@ def retrieve_trains_data( total_apps = len(apps) with concurrent.futures.ProcessPoolExecutor(max_workers=(5 if total_apps > 10 else 2)) as exc: for index, result in enumerate(zip(apps, exc.map( - functools.partial(app_details, apps, catalog_location, questions_context), + functools.partial( + app_details, apps, catalog_location, questions_context, normalize_questions=normalize_questions + ), apps, chunksize=(10 if total_apps > 10 else 5) ))): app_key = result[0]