Merge pull request #8 from truenas/library-validation

Enhance validation for defined libraries being used in apps

apps_validation/validation/app_version.py

@@ -11,6 +11,7 @@
 
 def validate_app_version_file(
     verrors: ValidationErrors, app_version_path: str, schema: str, item_name: str, version_name: Optional[str] = None,
+    train_name: Optional[str] = None,
 ) -> ValidationErrors:
     if os.path.exists(app_version_path):
         with open(app_version_path, 'r') as f:
@@ -64,6 +65,10 @@ def validate_app_version_file(
                             'Configured version in "app.yaml" does not match version directory name.'
                         )
 
+                    if train_name is not None:
+                        if app_config.get('train') != train_name:
+                            verrors.add(f'{schema}.train', 'Train name not correctly set in "app.yaml".')
+
     else:
         verrors.add(schema, 'Missing app version file')
 

apps_validation/validation/names.py

@@ -3,3 +3,4 @@
 
 RE_SCALE_VERSION = re.compile(r'^(\d{2}\.\d{2}(?:\.\d)*(?:-?(?:RC|BETA)\.?\d?)?)$')  # 24.04 / 24.04.1 / 24.04-RC.1
 RE_TRAIN_NAME = re.compile(r'^\w+[\w.-]*$')
+TEST_VALUES_FILENAME = 'test_values.yaml'

apps_validation/validation/validate_app.py

@@ -9,7 +9,7 @@
 from .utils import validate_key_value_types
 
 
-def validate_catalog_item(catalog_item_path: str, schema: str, validate_versions: bool = True):
+def validate_catalog_item(catalog_item_path: str, schema: str, train_name: str, validate_versions: bool = True):
     # We should ensure that each catalog item has at least 1 version available
     # Also that we have item.yaml present
     verrors = ValidationErrors()
@@ -57,7 +57,9 @@ def validate_catalog_item(catalog_item_path: str, schema: str, validate_versions
 
     for version_path in (versions if validate_versions else []):
         try:
-            validate_catalog_item_version(version_path, f'{schema}.versions.{os.path.basename(version_path)}')
+            validate_catalog_item_version(
+                version_path, f'{schema}.versions.{os.path.basename(version_path)}', train_name=train_name,
+            )
         except ValidationErrors as e:
             verrors.extend(e)
 

apps_validation/validation/validate_app_version.py

@@ -1,23 +1,28 @@
 import os
+import pathlib
 import typing
 import yaml
 
 from jsonschema import validate as json_schema_validate, ValidationError as JsonValidationError
 from semantic_version import Version
 
 from apps_validation.exceptions import ValidationErrors
+from catalog_reader.app_utils import get_app_basic_details
 from catalog_reader.questions_util import CUSTOM_PORTALS_KEY
 
 from .app_version import validate_app_version_file
 from .ix_values import validate_ix_values_schema
 from .json_schema_utils import METADATA_JSON_SCHEMA, VERSION_VALIDATION_SCHEMA
+from .names import TEST_VALUES_FILENAME
 from .validate_questions import validate_questions_yaml
+from .validate_templates import validate_templates
 
 
 WANTED_FILES_IN_ITEM_VERSION = {
     'app.yaml',
     'questions.yaml',
     'README.md',
+    TEST_VALUES_FILENAME,
 }
 
 
@@ -31,7 +36,7 @@ def validate_catalog_item_version_data(version_data: dict, schema: str, verrors:
 
 def validate_catalog_item_version(
     version_path: str, schema: str, version_name: typing.Optional[str] = None,
-    item_name: typing.Optional[str] = None, validate_values: bool = False,
+    item_name: typing.Optional[str] = None, validate_values: bool = False, train_name: typing.Optional[str] = None,
 ):
     verrors = ValidationErrors()
     version_name = version_name or os.path.basename(version_path)
@@ -48,7 +53,17 @@ def validate_catalog_item_version(
         verrors.add(f'{schema}.required_files', f'Missing {", ".join(files_diff)} required configuration files.')
 
     app_version_path = os.path.join(version_path, 'app.yaml')
-    validate_app_version_file(verrors, app_version_path, schema, item_name, version_name)
+    validate_app_version_file(verrors, app_version_path, schema, item_name, version_name, train_name=train_name)
+    app_basic_details = get_app_basic_details(version_path)
+    if app_basic_details.get('lib_version') is not None:
+        # Now we just want to make sure that actual directory for this lib version exists
+        if not pathlib.Path(
+            os.path.join(version_path, 'library', f'v{app_basic_details["lib_version"].replace(".", "_")}')
+        ).exists():
+            verrors.add(
+                f'{schema}.lib_version',
+                f'Specified {app_basic_details["lib_version"]!r} library version does not exist'
+            )
 
     questions_path = os.path.join(version_path, 'questions.yaml')
     if os.path.exists(questions_path):
@@ -57,8 +72,7 @@ def validate_catalog_item_version(
         except ValidationErrors as v:
             verrors.extend(v)
 
-    # FIXME: We should be validating templates as well
-    # FIXME: We should be validating specified functions as well
+    validate_templates(version_path, f'{schema}.templates')
 
     # FIXME: values.yaml is probably not needed here
     for values_file in ['ix_values.yaml'] + (['values.yaml'] if validate_values else []):

apps_validation/validation/validate_catalog.py

@@ -50,8 +50,6 @@ def validate_catalog(catalog_path: str):
         except ValidationErrors as e:
             verrors.extend(e)
 
-    # FIXME: Validate library structure and files
-    # FIXME: Validate ix-dev
     trains_dir = get_train_path(catalog_path)
     if not os.path.exists(trains_dir):
         verrors.add('trains', 'Trains directory is missing')
@@ -71,7 +69,7 @@ def validate_catalog(catalog_path: str):
 
     with concurrent.futures.ProcessPoolExecutor(max_workers=5 if len(items) > 10 else 2) as exc:
         for item in items:
-            item_futures.append(exc.submit(validate_catalog_item, item[0], item[1]))
+            item_futures.append(exc.submit(validate_catalog_item, item[0], item[1], item[2], True))
 
         for future in item_futures:
             try:

apps_validation/validation/validate_dev_directory.py

@@ -41,7 +41,7 @@ def validate_train(catalog_path: str, train_path: str, schema: str, to_check_app
 
         app_path = os.path.join(train_path, app_name)
         try:
-            validate_app(app_path, f'{schema}.{app_name}')
+            validate_app(app_path, f'{schema}.{app_name}', train_name)
         except ValidationErrors as ve:
             verrors.extend(ve)
         else:
@@ -65,14 +65,16 @@ def validate_upgrade_strategy(app_path: str, schema: str, verrors: ValidationErr
         verrors.add(schema, f'{upgrade_strategy_path!r} is not executable')
 
 
-def validate_app(app_dir_path: str, schema: str) -> None:
+def validate_app(app_dir_path: str, schema: str, train_name: str) -> None:
     app_name = os.path.basename(app_dir_path)
     chart_version_path = os.path.join(app_dir_path, 'app.yaml')
     verrors = validate_app_version_file(ValidationErrors(), chart_version_path, schema, app_name)
     validate_keep_versions(app_dir_path, app_name, verrors)
     verrors.check()
 
-    validate_catalog_item_version(app_dir_path, schema, get_app_version(app_dir_path), app_name, True)
+    validate_catalog_item_version(
+        app_dir_path, schema, get_app_version(app_dir_path), app_name, True, train_name=train_name,
+    )
 
     required_files = set(REQUIRED_METADATA_FILES)
     available_files = set(

apps_validation/validation/validate_templates.py

@@ -0,0 +1,78 @@
+import os
+import pathlib
+import re
+
+from apps_validation.exceptions import ValidationErrors
+from catalog_reader.app_utils import get_app_basic_details, get_values
+from catalog_templating.render import render_templates
+
+from .names import TEST_VALUES_FILENAME
+
+
+RE_APP_VERSION = re.compile(r'^v\d+_\d+_\d+$')
+RE_BASE_LIB_VERSION = re.compile(r'^base_v\d+_\d+_\d+$')
+
+
+def validate_templates(app_path: str, schema: str) -> None:
+    verrors = ValidationErrors()
+    templates_dir = pathlib.Path(os.path.join(app_path, 'templates'))
+    if not templates_dir.exists():
+        verrors.add(schema, 'Templates directory does not exist')
+    elif not templates_dir.is_dir():
+        verrors.add(schema, 'Templates is not a directory')
+    else:
+        if (library_path := templates_dir / 'library').exists():
+            if library_path.exists():
+                if not library_path.is_dir():
+                    verrors.add(f'{schema}.library', 'Library is not a directory')
+                else:
+                    validate_library(app_path, f'{schema}.library', verrors)
+
+        found_compose_file = False
+        for entry in templates_dir.iterdir():
+            if entry.name.endswith('.yaml'):
+                if not entry.is_file():
+                    verrors.add(schema, f'{entry.name!r} template file is not a file')
+                else:
+                    found_compose_file = True
+
+        if not found_compose_file:
+            verrors.add(schema, 'No template files found in templates directory')
+
+    verrors.check()
+
+
+def validate_library(app_path: str, schema: str, verrors: ValidationErrors) -> None:
+    library_dir = pathlib.Path(os.path.join(app_path, 'templates/library'))
+    library_contents = list(library_dir.iterdir())
+    if not library_contents:
+        return
+    elif len(library_contents) > 2:
+        verrors.add(schema, 'Library directory should only contain library version from the catalog or the app')
+
+    app_config = get_app_basic_details(app_path)
+    app_library = pathlib.Path(os.path.join(
+        library_dir.name, app_config['train'], app_config['name'], f'v{app_config["version"].replace(".", "_")}'
+    ))
+    # We expect 2 paths here, one for the base library and one for the app library
+    for entry in library_contents:
+        if RE_BASE_LIB_VERSION.findall(entry.name):
+            if not entry.is_dir():
+                verrors.add(schema, f'Base library {entry.name!r} is not a directory')
+        elif entry.name == app_config['train']:
+            if app_library.exists() and not app_library.is_dir():
+                verrors.add(schema, f'App library {app_library.name!r} is not a directory')
+        else:
+            verrors.add(schema, f'Unexpected library found: {entry.name!r}')
+
+    if verrors:
+        # If we have issues, no point in continuing further
+        return
+
+    try:
+        rendered = render_templates(app_path, get_values(os.path.join(app_path, TEST_VALUES_FILENAME)))
+    except Exception as e:
+        verrors.add(schema, f'Failed to render templates: {e}')
+    else:
+        if not rendered:
+            verrors.add(schema, 'No templates were rendered')

apps_validation/validation/validate_train.py

@@ -22,5 +22,5 @@ def get_train_items(train_path: str) -> typing.List[typing.Tuple[str, str]]:
         item_path = os.path.join(train_path, catalog_item)
         if not os.path.isdir(item_path):
             continue
-        items.append((item_path, f'trains.{train}.{catalog_item}'))
+        items.append((item_path, f'trains.{train}.{catalog_item}', train))
     return items

catalog_reader/app.py

@@ -37,7 +37,7 @@ def get_app_details(
 
     schema = f'{train}.{item}'
     try:
-        validate_catalog_item(item_location, schema, False)
+        validate_catalog_item(item_location, schema, train, False)
     except ValidationErrors as verrors:
         item_data['healthy_error'] = f'Following error(s) were found with {item!r}:\n'
         for verror in verrors:

catalog_reader/app_utils.py

@@ -47,7 +47,9 @@ def get_app_basic_details(app_path: str) -> dict:
     with contextlib.suppress(FileNotFoundError, yaml.YAMLError, KeyError):
         with open(os.path.join(app_path, 'app.yaml'), 'r') as f:
             app_config = yaml.safe_load(f.read())
-        return {k: app_config[k] for k in ('name', 'train', 'version', 'lib_version')}
+        return {'lib_version': app_config.get('lib_version')} | {
+            k: app_config[k] for k in ('name', 'train', 'version')
+        }
 
     return {}
 

catalog_templating/render.py

@@ -38,11 +38,11 @@ def render_templates(app_version_path: str, test_values: dict) -> dict:
 def import_library(library_path: str, app_config) -> dict:
     modules_context = collections.defaultdict(dict)
     # 2 dirs which we want to import from
-    global_base_lib = os.path.join(library_path, f'base_v{app_config["lib_version"].replace(".", "_")}')
+    global_base_lib = os.path.join(library_path, f'base_v{(app_config["lib_version"] or "").replace(".", "_")}')
     app_lib = os.path.join(
         library_path, app_config['train'], app_config['name'], f'v{app_config["version"].replace(".", "_")}'
     )
-    if pathlib.Path(global_base_lib).is_dir():
+    if app_config['lib_version'] and pathlib.Path(global_base_lib).is_dir():
         modules_context['base'] = import_app_modules(global_base_lib, os.path.basename(global_base_lib))  # base_v1_0_0
     if pathlib.Path(app_lib).is_dir():
         modules_context[app_config['train']] = {