From 38776498dcaa0bee51f8a939672fda6c04ab5f1a Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Mon, 9 Dec 2024 17:06:21 +0100
Subject: [PATCH 01/13] validate URL query parameters and include/exclude
 patterns starting with .*

---
 .github/sc4pac-yaml-schema.py | 56 +++++++++++++++++++++++++----------
 src/yaml/jenx/fauna.yaml      |  2 +-
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 3d3fc767..5ad4de94 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -6,6 +6,9 @@
 import sys
 import os
 import re
+from urllib.parse import (urlparse, parse_qs)
+import jsonschema
+from jsonschema import ValidationError
 
 # add subfolders as necessary
 subfolders = r"""
@@ -68,7 +71,7 @@
         "assetId": {"type": "string"},
         "version": {"type": "string"},
         "lastModified": {"type": "string"},
-        "url": {"type": "string"},
+        "url": {"type": "string", "validate_query_params": True},
         "archiveType": {
             "type": "object",
             "additionalProperties": False,
@@ -88,8 +91,8 @@
         "required": ["assetId"],
         "properties": {
             "assetId": {"type": "string"},
-            "include": unique_strings,
-            "exclude": unique_strings,
+            "include": {**unique_strings, "validate_pattern": True},
+            "exclude": {**unique_strings, "validate_pattern": True},
         },
     },
 }
@@ -133,7 +136,7 @@
                 "description": {"type": "string"},
                 "author": {"type": "string"},
                 "images": unique_strings,
-                "website": {"type": "string"},
+                "website": {"type": "string", "validate_query_params": True},
             },
         },
     },
@@ -323,15 +326,45 @@ def validate_document_separators(text) -> None:
                 "YAML file contains multiple package and asset definitions. They all need to be separated by `---`.")
 
 
+def validate_pattern(validator, value, instance, schema):
+    patterns = [instance] if isinstance(instance, str) else instance
+    bad_patterns = [p for p in patterns if p.startswith('.*')]
+    if bad_patterns:
+        yield ValidationError(f"include/exclude patterns should not start with '.*' in {bad_patterns}")
+
+
+_irrelevant_query_parameters = [
+    ("sc4evermore.com", ("catid",)),
+    ("simtropolis.com", ("confirm", "t", "csrfKey")),
+]
+
+
+def validate_query_params(validator, value, url, schema):
+    msgs = []
+    if '/sc4evermore.com/' in url:
+        msgs.append(f"Domain of URL {url} should be www.sc4evermore.com (add www.)")
+    qs = parse_qs(urlparse(url).query)
+    bad_params = [p for domain, params in _irrelevant_query_parameters
+                  if domain in url for p in params if p in qs]
+    if bad_params:
+        msgs.append(f"Avoid these URL query parameters: {', '.join(bad_params)}")
+    if msgs:
+        yield ValidationError('\n'.join(msgs))
+
+
 def main() -> int:
     args = sys.argv[1:]
     if not args:
         "Pass at least one directory or yaml file to validate as argument."
         return 1
 
-    from jsonschema.validators import Draft202012Validator
-    from jsonschema import exceptions
-    validator = Draft202012Validator(schema)
+    validator = jsonschema.validators.extend(
+            jsonschema.validators.Draft202012Validator,
+            validators=dict(
+                validate_pattern=validate_pattern,
+                validate_query_params=validate_query_params,
+            ),
+        )(schema)
     validator.check_schema(schema)
     dependency_checker = DependencyChecker()
     validated = 0
@@ -351,16 +384,9 @@ def main() -> int:
                             if doc is None:  # empty yaml file or document
                                 continue
                             dependency_checker.aggregate_identifiers(doc)
-                            err = exceptions.best_match(validator.iter_errors(doc))
+                            err = jsonschema.exceptions.best_match(validator.iter_errors(doc))
                             msgs = [] if err is None else [err.message]
 
-                            # check URLs
-                            urls = [u for u in [doc.get('url'), doc.get('info', {}).get('website')]
-                                    if u is not None]
-                            for u in urls:
-                                if '/sc4evermore.com/' in u:
-                                    msgs.append(f"Domain of URL {u} should be www.sc4evermore.com")
-
                             # check "None" value
                             for label in ['conflicts', 'warning', 'summary', 'description']:
                                 field = doc.get('info', {}).get(label)
diff --git a/src/yaml/jenx/fauna.yaml b/src/yaml/jenx/fauna.yaml
index 070955c3..be3c3c33 100644
--- a/src/yaml/jenx/fauna.yaml
+++ b/src/yaml/jenx/fauna.yaml
@@ -2,7 +2,7 @@
 assetId: "jenx-fauna"
 version: "1.0"
 lastModified: "2023-09-15T16:33:15-07:00"
-url: "https://www.sc4evermore.com/index.php/downloads?task=download.send&id=155:sc4d-lex-legacy-jenx-fauna&catid=25"
+url: "https://www.sc4evermore.com/index.php/downloads?task=download.send&id=155:sc4d-lex-legacy-jenx-fauna"
 
 ---
 group: "jenx"

From 95c48bc8b011f18872399b42c49dec845ca20b78 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Mon, 9 Dec 2024 20:34:42 +0100
Subject: [PATCH 02/13] refactor validation of text fields

---
 .github/sc4pac-yaml-schema.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 5ad4de94..5eee0ee4 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -131,9 +131,9 @@
             "additionalProperties": False,
             "properties": {
                 "summary": {"type": "string"},
-                "warning": {"type": "string"},
-                "conflicts": {"type": "string"},
-                "description": {"type": "string"},
+                "warning": {"type": "string", "validate_text_field": "warning"},
+                "conflicts": {"type": "string", "validate_text_field": "conflicts"},
+                "description": {"type": "string", "validate_text_field": "description"},
                 "author": {"type": "string"},
                 "images": unique_strings,
                 "website": {"type": "string", "validate_query_params": True},
@@ -352,6 +352,18 @@ def validate_query_params(validator, value, url, schema):
         yield ValidationError('\n'.join(msgs))
 
 
+def validate_text_field(validator, field, text, schema):
+    msgs = []
+    if text is not None and text.strip().lower() == "none":
+        msgs.append(f"""Text "{field}" should not be "{text.strip()}", but should be omitted instead.""")
+    if text is not None and DependencyChecker.pronouns_pattern.search(text):
+        msgs.append(f"""The "{field}" should be written in a neutral perspective (avoid the words 'I', 'me', 'my').""")
+    if text is not None and DependencyChecker.desc_invalid_chars_pattern.search(text):
+        msgs.append("""The "{field}" seems to be malformed (avoid the characters '\\n', '\\"').""")
+    if msgs:
+        yield ValidationError('\n'.join(msgs))
+
+
 def main() -> int:
     args = sys.argv[1:]
     if not args:
@@ -363,6 +375,7 @@ def main() -> int:
             validators=dict(
                 validate_pattern=validate_pattern,
                 validate_query_params=validate_query_params,
+                validate_text_field=validate_text_field,
             ),
         )(schema)
     validator.check_schema(schema)
@@ -387,18 +400,6 @@ def main() -> int:
                             err = jsonschema.exceptions.best_match(validator.iter_errors(doc))
                             msgs = [] if err is None else [err.message]
 
-                            # check "None" value
-                            for label in ['conflicts', 'warning', 'summary', 'description']:
-                                field = doc.get('info', {}).get(label)
-                                if field is not None and field.strip().lower() == "none":
-                                    msgs.append(f"""Field "{label}" should not be "{field.strip()}", but should be left out instead.""")
-
-                            desc = doc.get('info', {}).get('description')
-                            if desc is not None and dependency_checker.pronouns_pattern.search(desc):
-                                msgs.append("The description should be written in a neutral perspective (avoid the words 'I', 'me', 'my').")
-                            if desc is not None and dependency_checker.desc_invalid_chars_pattern.search(desc):
-                                msgs.append("""The description seems to be malformed (avoid the characters '\\n', '\\"').""")
-
                             if msgs:
                                 errors += 1
                                 print(f"===> {p}")

From 08d79a10eeab0e229fb5b54360e6dc7aec7deed5 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 16:25:59 +0100
Subject: [PATCH 03/13] add support for checksum and withChecksum fields

---
 .github/sc4pac-yaml-schema.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 5eee0ee4..7174b160 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -80,6 +80,14 @@
                 "version": {"enum": ["20", "24", "30", "35", "40"]},
             },
         },
+        "checksum": {
+            "type": "object",
+            "additionalProperties": False,
+            "required": ["sha256"],
+            "properties": {
+                "sha256": {"type": "string", "validate_sha256": True},
+            },
+        },
     },
 }
 
@@ -93,6 +101,19 @@
             "assetId": {"type": "string"},
             "include": {**unique_strings, "validate_pattern": True},
             "exclude": {**unique_strings, "validate_pattern": True},
+            "withChecksum": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "required": ["include", "sha256"],
+                    "properties": {
+                        "include": {"type": "string", "validate_pattern": True},
+                        "sha256": {"type": "string", "validate_sha256": True},
+                    },
+                },
+                "uniqueItems": True,
+            },
         },
     },
 }
@@ -161,6 +182,7 @@ class DependencyChecker:
     version_rel_pattern = re.compile(r"(.*?)(-\d+)?")
     pronouns_pattern = re.compile(r"\b[Mm][ey]\b|(?:\bI\b(?!-|\.| [A-Z]))")
     desc_invalid_chars_pattern = re.compile(r'\\n|\\"')
+    sha256_pattern = re.compile(r"[a-f0-9]*", re.IGNORECASE)
 
     def __init__(self):
         self.known_packages = set()
@@ -364,6 +386,11 @@ def validate_text_field(validator, field, text, schema):
         yield ValidationError('\n'.join(msgs))
 
 
+def validate_sha256(validator, value, text, schema):
+    if not (len(text) == 64 and DependencyChecker.sha256_pattern.fullmatch(text)):
+        yield ValidationError(f"value is not a sha256: {text}")
+
+
 def main() -> int:
     args = sys.argv[1:]
     if not args:
@@ -376,6 +403,7 @@ def main() -> int:
                 validate_pattern=validate_pattern,
                 validate_query_params=validate_query_params,
                 validate_text_field=validate_text_field,
+                validate_sha256=validate_sha256,
             ),
         )(schema)
     validator.check_schema(schema)

From b5b8f7076009848555046174a5bbb5ae84cbe708 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 17:34:53 +0100
Subject: [PATCH 04/13] check package names containing 'vol'

---
 .github/sc4pac-yaml-schema.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 7174b160..31ab1415 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -125,7 +125,7 @@
     "required": ["group", "name", "version", "subfolder"],
     "properties": {
         "group": {"type": "string"},
-        "name": {"type": "string"},
+        "name": {"type": "string", "validate_name": True},
         "version": {"type": "string"},
         "subfolder": {"enum": subfolders},
         "dependencies": unique_strings,
@@ -374,6 +374,11 @@ def validate_query_params(validator, value, url, schema):
         yield ValidationError('\n'.join(msgs))
 
 
+def validate_name(validator, value, name, schema):
+    if "-vol-" in name:
+        yield ValidationError(f"Avoid the hyphen after 'vol' (for consistency with other packages): {name}")
+
+
 def validate_text_field(validator, field, text, schema):
     msgs = []
     if text is not None and text.strip().lower() == "none":
@@ -402,6 +407,7 @@ def main() -> int:
             validators=dict(
                 validate_pattern=validate_pattern,
                 validate_query_params=validate_query_params,
+                validate_name=validate_name,
                 validate_text_field=validate_text_field,
                 validate_sha256=validate_sha256,
             ),

From e0089e15262c6ebca7fa4a3b8e5306aad1c17944 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:01:46 +0100
Subject: [PATCH 05/13] add check for DLLs without checksum

---
 .github/sc4pac-yaml-schema.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 31ab1415..702543ab 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -203,6 +203,7 @@ def __init__(self):
         self.invalid_variant_names = set()
         self.packages_with_single_assets = {}  # pkg -> (version, set of assets from variants)
         self.packages_using_asset = {}  # asset -> set of packages
+        self.dlls_without_checksum = set()
 
     def aggregate_identifiers(self, doc):
         if 'assetId' in doc:
@@ -229,7 +230,12 @@ def aggregate_identifiers(self, doc):
             def asset_ids(obj):
                 return (a['assetId'] for a in obj.get('assets', []) if 'assetId' in a)
 
-            def add_references(obj):
+            variants0 = doc.get('variants', [])
+            def iterate_doc_and_variants():
+                yield doc
+                yield from variants0
+
+            for obj in iterate_doc_and_variants():
                 local_deps = obj.get('dependencies', [])
                 self.referenced_packages.update(local_deps)
                 if pkg in local_deps:
@@ -242,11 +248,6 @@ def add_references(obj):
                     else:
                         self.packages_using_asset[a] = set([pkg])
 
-            variants0 = doc.get('variants', [])
-            add_references(doc)
-            for v in variants0:
-                add_references(v)
-
             num_doc_assets = len(doc.get('assets', []))
             if num_doc_assets <= 1:
                 single_assets = set(asset_ids(doc))
@@ -275,6 +276,19 @@ def add_references(obj):
                     if not self.naming_convention_variants_value.fullmatch(value):
                         self.invalid_variant_names.add(value)
 
+            is_dll = ("DLL" in doc.get('info', {}).get('summary', "")) or ("dll" in doc['name'].split('-'))
+            if is_dll:
+                has_asset = False
+                has_checksum = False
+                for obj in iterate_doc_and_variants():
+                    for asset in obj.get('assets', []):
+                        has_asset = True
+                        if "withChecksum" in asset:
+                            has_checksum = True
+                if has_asset and not has_checksum:
+                    self.dlls_without_checksum.add(pkg)
+
+
     def _get_channel_contents(self, channel_url):
         import urllib.request
         import json
@@ -521,6 +535,12 @@ def main() -> int:
             for pkg, v1, asset, v2 in version_mismatches:
                 print(f"""{pkg} "{v1}" (expected version "{v2}" of asset {asset})""")
 
+        if dependency_checker.dlls_without_checksum:
+            errors += len(dependency_checker.dlls_without_checksum)
+            print("===> The following packages appear to contain DLLs. A sha256 checksum is required for DLLs (add a `withChecksum` field).")
+            for identifier in dependency_checker.dlls_without_checksum:
+                print(identifier)
+
     if errors > 0:
         print(f"Finished with {errors} errors.")
         return 1

From 4b0995c243970540488b61897ef9a7c9220a8e46 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:44:53 +0100
Subject: [PATCH 06/13] add support for nonPersistentUrl field for assets

---
 .github/sc4e-check-updates.py | 2 +-
 .github/sc4pac-yaml-schema.py | 1 +
 .github/st-check-updates.py   | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/sc4e-check-updates.py b/.github/sc4e-check-updates.py
index 8fae46db..b98e63b3 100644
--- a/.github/sc4e-check-updates.py
+++ b/.github/sc4e-check-updates.py
@@ -56,7 +56,7 @@ def main() -> int:
             continue
 
         # check URLs
-        url = doc.get('url')
+        url = doc.get('nonPersistentUrl') or doc.get('url')
         if url is None:
             continue  # not an asset
         m = url_id_pattern.fullmatch(url)
diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 702543ab..dbb09ded 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -72,6 +72,7 @@
         "version": {"type": "string"},
         "lastModified": {"type": "string"},
         "url": {"type": "string", "validate_query_params": True},
+        "nonPersistentUrl": {"type": "string", "validate_query_params": True},
         "archiveType": {
             "type": "object",
             "additionalProperties": False,
diff --git a/.github/st-check-updates.py b/.github/st-check-updates.py
index e7da662c..c9367818 100644
--- a/.github/st-check-updates.py
+++ b/.github/st-check-updates.py
@@ -65,7 +65,7 @@ def main() -> int:
                 continue
 
             # find all STEX file IDs
-            url = doc.get('url')
+            url = doc.get('nonPersistentUrl') or doc.get('url')
             if url is None:
                 continue  # not an asset
             m = url_id_pattern.fullmatch(url)
@@ -98,7 +98,7 @@ def main() -> int:
             continue
 
         # check URLs
-        url = doc.get('url')
+        url = doc.get('nonPersistentUrl') or doc.get('url')
         if url is None:
             continue  # not an asset
         m = url_id_pattern.fullmatch(url)

From 747a8a78b947f23ee4c24e9bb106299f953734ba Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:08:15 +0100
Subject: [PATCH 07/13] add check for http urls without checksum

---
 .github/sc4pac-yaml-schema.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index dbb09ded..69f01785 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -205,6 +205,7 @@ def __init__(self):
         self.packages_with_single_assets = {}  # pkg -> (version, set of assets from variants)
         self.packages_using_asset = {}  # asset -> set of packages
         self.dlls_without_checksum = set()
+        self.http_without_checksum = set()
 
     def aggregate_identifiers(self, doc):
         if 'assetId' in doc:
@@ -213,10 +214,13 @@ def aggregate_identifiers(self, doc):
                 self.known_assets.add(asset)
             else:
                 self.duplicate_assets.add(asset)
-            self.asset_urls[asset] = doc.get('url')
+            url = doc.get('url')
+            self.asset_urls[asset] = url
             self.asset_versions[asset] = doc.get('version')
             if not self.naming_convention.fullmatch(asset):
                 self.invalid_asset_names.add(asset)
+            if urlparse(url).scheme not in ['https', 'file'] and 'checksum' not in doc:
+                self.http_without_checksum.add(asset)
         if 'group' in doc and 'name' in doc:
             pkg = doc['group'] + ":" + doc['name']
             if pkg not in self.known_packages:
@@ -542,6 +546,12 @@ def main() -> int:
             for identifier in dependency_checker.dlls_without_checksum:
                 print(identifier)
 
+        if dependency_checker.http_without_checksum:
+            errors += len(dependency_checker.http_without_checksum)
+            print("===> The following assets use http instead of https. They should include a `checksum` field.")
+            for identifier in dependency_checker.http_without_checksum:
+                print(identifier)
+
     if errors > 0:
         print(f"Finished with {errors} errors.")
         return 1

From 003fb915ef3e9f05ba10d40993b8198fbdbf1a2e Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Tue, 10 Dec 2024 19:53:05 +0100
Subject: [PATCH 08/13] refactoring: condense error reporting format

---
 .github/sc4pac-yaml-schema.py | 111 +++++++++-------------------------
 1 file changed, 28 insertions(+), 83 deletions(-)

diff --git a/.github/sc4pac-yaml-schema.py b/.github/sc4pac-yaml-schema.py
index 69f01785..e2d23775 100644
--- a/.github/sc4pac-yaml-schema.py
+++ b/.github/sc4pac-yaml-schema.py
@@ -435,6 +435,15 @@ def main() -> int:
     dependency_checker = DependencyChecker()
     validated = 0
     errors = 0
+
+    def basic_report(identifiers, msg: str, stringify=None):
+        if identifiers:
+            nonlocal errors
+            errors += len(identifiers)
+            print(f"===> {msg}")
+            for identifier in identifiers:
+                print(identifier if stringify is None else stringify(identifier))
+
     for d in args:
         for (root, dirs, files) in os.walk(d):
             for fname in files:
@@ -467,90 +476,26 @@ def main() -> int:
         # check that all dependencies exist
         # (this check only makes sense for the self-contained main channel)
         for label, unknown in dependency_checker.unknowns().items():
-            if unknown:
-                errors += len(unknown)
-                print(f"===> The following {label} are referenced, but not defined:")
-                for identifier in unknown:
-                    print(identifier)
-
+            basic_report(unknown, f"The following {label} are referenced, but not defined:")
         for label, dupes in dependency_checker.duplicates().items():
-            if dupes:
-                errors += len(dupes)
-                print(f"===> The following {label} are defined multiple times:")
-                for identifier in dupes:
-                    print(identifier)
-
-        if dependency_checker.self_dependencies:
-            errors += len(dependency_checker.self_dependencies)
-            print("===> The following packages unnecessarily depend on themselves:")
-            for pkg in dependency_checker.self_dependencies:
-                print(pkg)
-
-        non_unique_assets = dependency_checker.assets_with_same_url()
-        if non_unique_assets:
-            errors += len(non_unique_assets)
-            print("===> The following assets have the same URL (The same asset was defined twice with different asset IDs):")
-            for assets in non_unique_assets:
-                print(', '.join(assets))
-
-        unused_assets = dependency_checker.unused_assets()
-        if unused_assets:
-            errors += len(unused_assets)
-            print("===> The following assets are not used:")
-            for identifier in unused_assets:
-                print(identifier)
-
-        if dependency_checker.overlapping_variants:
-            errors += len(dependency_checker.overlapping_variants)
-            print("===> The following packages have duplicate variants:")
-            for pkg in dependency_checker.overlapping_variants:
-                print(pkg)
-
-        if dependency_checker.unexpected_variants:
-            errors += len(dependency_checker.unexpected_variants)
-            print("===>")
-            for pkg, key, values, expected_values in dependency_checker.unexpected_variants:
-                print(f"{pkg} defines {key} variants {values} (expected: {expected_values})")
-
-        if dependency_checker.invalid_asset_names:
-            errors += len(dependency_checker.invalid_asset_names)
-            print("===> the following assetIds do not match the naming convention (lowercase alphanumeric hyphenated)")
-            for identifier in dependency_checker.invalid_asset_names:
-                print(identifier)
-        if dependency_checker.invalid_group_names:
-            errors += len(dependency_checker.invalid_group_names)
-            print("===> the following group identifiers do not match the naming convention (lowercase alphanumeric hyphenated)")
-            for identifier in dependency_checker.invalid_group_names:
-                print(identifier)
-        if dependency_checker.invalid_package_names:
-            errors += len(dependency_checker.invalid_package_names)
-            print("===> the following package names do not match the naming convention (lowercase alphanumeric hyphenated)")
-            for identifier in dependency_checker.invalid_package_names:
-                print(identifier)
-        if dependency_checker.invalid_variant_names:
-            errors += len(dependency_checker.invalid_variant_names)
-            print("===> the following variant labels or values do not match the naming convention (alphanumeric hyphenated or dots)")
-            for identifier in dependency_checker.invalid_variant_names:
-                print(identifier)
-
-        version_mismatches = list(dependency_checker.package_asset_version_mismatches())
-        if version_mismatches:
-            errors += len(version_mismatches)
-            print("===> The versions of the following packages do not match the version of the referenced assets (usually they should agree, but if the version mismatch is intentional, the packages can be added to the ignore list in .github/sc4pac-yaml-schema.py):")
-            for pkg, v1, asset, v2 in version_mismatches:
-                print(f"""{pkg} "{v1}" (expected version "{v2}" of asset {asset})""")
-
-        if dependency_checker.dlls_without_checksum:
-            errors += len(dependency_checker.dlls_without_checksum)
-            print("===> The following packages appear to contain DLLs. A sha256 checksum is required for DLLs (add a `withChecksum` field).")
-            for identifier in dependency_checker.dlls_without_checksum:
-                print(identifier)
-
-        if dependency_checker.http_without_checksum:
-            errors += len(dependency_checker.http_without_checksum)
-            print("===> The following assets use http instead of https. They should include a `checksum` field.")
-            for identifier in dependency_checker.http_without_checksum:
-                print(identifier)
+            basic_report(dupes, f"The following {label} are defined multiple times:")
+        basic_report(dependency_checker.self_dependencies, "The following packages unnecessarily depend on themselves:")
+        basic_report(dependency_checker.assets_with_same_url(),
+                     "The following assets have the same URL (The same asset was defined twice with different asset IDs):",
+                     lambda assets: ', '.join(assets))
+        basic_report(dependency_checker.unused_assets(), "The following assets are not used:")
+        basic_report(dependency_checker.overlapping_variants, "The following packages have duplicate variants:")
+        basic_report(dependency_checker.unexpected_variants, "",
+                     lambda tup: "{0} defines unexpected {1} variants {2} (expected: {3})".format(*tup))  # pkg, key, values, expected_values
+        basic_report(dependency_checker.invalid_asset_names, "the following assetIds do not match the naming convention (lowercase alphanumeric hyphenated)")
+        basic_report(dependency_checker.invalid_group_names, "the following group identifiers do not match the naming convention (lowercase alphanumeric hyphenated)")
+        basic_report(dependency_checker.invalid_package_names, "the following package names do not match the naming convention (lowercase alphanumeric hyphenated)")
+        basic_report(dependency_checker.invalid_variant_names, "the following variant labels or values do not match the naming convention (alphanumeric hyphenated or dots)")
+        basic_report(list(dependency_checker.package_asset_version_mismatches()),
+                     "The versions of the following packages do not match the version of the referenced assets (usually they should agree, but if the version mismatch is intentional, the packages can be added to the ignore list in .github/sc4pac-yaml-schema.py):",
+                     lambda tup: """{0} "{1}" (expected version "{3}" of asset {2})""".format(*tup))  # pkg, v1, asset, v2
+        basic_report(dependency_checker.dlls_without_checksum, "The following packages appear to contain DLLs. A sha256 checksum is required for DLLs (add a `withChecksum` field).")
+        basic_report(dependency_checker.http_without_checksum, "The following assets use http instead of https. They should include a `checksum` field.")
 
     if errors > 0:
         print(f"Finished with {errors} errors.")

From 47f493ae079405f622f28332dbb0599ae3707650 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:04:42 +0100
Subject: [PATCH 09/13] update metadata documentation for checksums and
 nonPersistentUrl

---
 docs/_sidebar.md |  2 +-
 docs/metadata.md | 51 +++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/docs/_sidebar.md b/docs/_sidebar.md
index 79633596..b424854f 100644
--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -2,7 +2,7 @@
   * [Getting started](/)
   * [CLI](cli.md)
   * [API](api.md)
-  * [Adding metadata](metadata.md)
+  * [Metadata format](metadata.md)
   * [About](about.md)
 - Packages
   * [Highlights](packages.md)
diff --git a/docs/metadata.md b/docs/metadata.md
index 5f03a060..f622882d 100644
--- a/docs/metadata.md
+++ b/docs/metadata.md
@@ -1,4 +1,4 @@
-# Adding metadata
+# Metadata format
 
 This page details how to write, for an existing mod, custom metadata that is understood by *sc4pac*.
 The metadata is stored in [YAML](https://en.wikipedia.org/wiki/YAML) files which can be edited in any text editor
@@ -83,6 +83,31 @@ On SC4Evermore, grab the *Changed* timestamp from the info box on the download p
 For other sites, use the available info on the download page or, when supported by the server,
 use the `Last-Modified` HTTP header of the download URL. Shorthand for cURL users: `curl -I -L '<url>'`.
 
+### `checksum`
+
+An optional sha256 checksum can be added for verifying file integrity.
+If present, the checksum of the file is checked directly after download, before extracting the archive.
+```yaml
+checksum:
+  sha256: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+```
+This is recommended for files that are downloaded using http instead of https.
+
+### `nonPersistentUrl`
+
+When the `url` points to a specific _persistent_ version of a file,
+a `nonPersistentUrl` can be added that always points to the _latest_ version of the file.
+```yaml
+nonPersistentUrl: "https://community.simtropolis.com/files/file/25137-hogwarts-castle/?do=download"
+url: "https://github.com/....../v950/hogwarts-castle.zip"
+```
+
+?> Mainly, this is useful for DLLs released on GitHub and Simtropolis.
+   The `url` would point to the current release on GitHub, while the `nonPersistentUrl` links to Simtropolis.
+   The `nonPersistentUrl` is never used for downloading, but can be used by tools to check for updates.
+   As the file is downloaded from a specific release on GitHub,
+   this avoids intermittent checksum errors when the metadata has not yet been updated after a new release has been uploaded to Simtropolis.
+
 ### `archiveType`
 
 This is only needed for assets containing Clickteam exe-installers. It is not needed for NSIS exe-installers.
@@ -236,12 +261,32 @@ Details:
 - The matching is case-insensitive for file-system independence.
 - In any case, always both `include` and `exclude` filters are evaluated.
   If one or both are absent, they default to the following behavior:
-  - If the `include` filter is absent or empty, then by default all files with file type .dat/.sc4model/.sc4lot/.sc4desc/.sc4/.dll are included.
-  - If the `exclude` filter is absent or empty, then by default all file types other than .dat/.sc4model/.sc4lot/.sc4desc/.sc4/.dll are excluded.
+  - If the `include` filter is absent or empty, then by default all files with file type .dat/.sc4model/.sc4lot/.sc4desc/.sc4 are included.
+  - If the `exclude` filter is absent or empty, then by default all file types other than .dat/.sc4model/.sc4lot/.sc4desc/.sc4 are excluded.
+- All extracted files without checksum must be DBPF files.
 
 ?> If you anticipate file names changing with future updates of the original upload,
    consider using regular expressions to make the matching more generic, so that the `include` filter keeps working after the updates.
 
+### `withChecksum`
+
+In addition to the `include`/`exclude` fields, you can use a `withChecksum` block to only include files if their checksum matches.
+This is _required_ for DLL files (code mods) and other non-DBPF file types.
+The checksums are verified after the files are extracted from an asset,
+but before they are moved from a temporary location into the plugins folder loaded by the game.
+```yaml
+assets:
+- assetId: "dumbledore-hogwarts-castle"
+  include:
+  - "/Hogwarts/"  # only DBPF files
+  withChecksum:
+  - include: "/magic.dll"
+    sha256: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+```
+In contrast to the `checksum` field of an asset, this is the sha256 hash of the extracted file itself (e.g. hash of the .dll instead of the .zip file).
+
+?> When using `withChecksum`, it is recommended to also add a [`nonPersistentUrl`](#nonPersistentUrl) to the corresponding asset definition.
+
 ### `info`
 
 Additional descriptive information.

From f682cae3ec8d5a1b4c12a9f5393ca26274661b3a Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Thu, 12 Dec 2024 07:34:46 +0100
Subject: [PATCH 10/13] add checksums to metadata

---
 src/yaml/aqua877/jrs-prop-pack.yaml           |  4 ++-
 src/yaml/jim/carprop-pack.yaml                |  4 ++-
 src/yaml/memo/3d-camera-dll.yaml              | 10 ++++--
 src/yaml/memo/region-thumbnail-fix-dll.yaml   | 10 ++++--
 src/yaml/memo/submenus-dll.yaml               | 10 ++++--
 .../memo/transparent-texture-fix-dll.yaml     | 10 ++++--
 src/yaml/neko/prop-sets.yaml                  | 32 ++++++++++++++-----
 .../null-45/sc4-resource-loading-hooks.yaml   | 14 +++++---
 src/yaml/simmaster07/extra-cheats-dll.yaml    |  8 ++++-
 src/yaml/simmaster07/sc4fix.yaml              | 12 ++++---
 10 files changed, 82 insertions(+), 32 deletions(-)

diff --git a/src/yaml/aqua877/jrs-prop-pack.yaml b/src/yaml/aqua877/jrs-prop-pack.yaml
index 17bb8eef..e564dc77 100644
--- a/src/yaml/aqua877/jrs-prop-pack.yaml
+++ b/src/yaml/aqua877/jrs-prop-pack.yaml
@@ -11,6 +11,8 @@ info:
 
 ---
 assetId: "aqua-877-jrs-prop-pack-signs"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:37:13Z"
 url: "http://hide-inoki.com/bbs/archives/files/1107.zip"
+checksum:
+  sha256: 28cbea30c01afe556884236210d0557c68787acc84c93535b9878aba10635ee3
diff --git a/src/yaml/jim/carprop-pack.yaml b/src/yaml/jim/carprop-pack.yaml
index 6a437184..cf5977cc 100644
--- a/src/yaml/jim/carprop-pack.yaml
+++ b/src/yaml/jim/carprop-pack.yaml
@@ -13,6 +13,8 @@ assets:
 
 ---
 assetId: hide-inoki-jim-carprop
-version: "1.2"
+version: "1.2-1"
 lastModified: "2012-11-11T23:17:00Z"
 url: http://hide-inoki.com/bbs/archives/files/1598.zip
+checksum:
+  sha256: d1a4e2735436867e46b847a83266874fe4963517d88f147dc795bdb9b4862d1c
diff --git a/src/yaml/memo/3d-camera-dll.yaml b/src/yaml/memo/3d-camera-dll.yaml
index e47e62e0..6ee52710 100644
--- a/src/yaml/memo/3d-camera-dll.yaml
+++ b/src/yaml/memo/3d-camera-dll.yaml
@@ -1,9 +1,12 @@
 group: "memo"
 name: "3d-camera-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-3d-camera-dll"
+  withChecksum:
+  - include: "/memo.3dcamera.dll"
+    sha256: 281406de45ea19ebd886b2584d2417781cca6ea874973cc4bd4b1f2e9d5ee216
 
 info:
   summary: "Allows setting arbitrary camera angles"
@@ -29,6 +32,7 @@ info:
 
 ---
 assetId: "memo-3d-camera-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-04-01T08:26:25Z"
-url: "https://community.simtropolis.com/files/file/36188-3d-camera-dll-for-simcity-4/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36188-3d-camera-dll-for-simcity-4/?do=download"
+url: "https://github.com/memo33/sc4-3d-camera-dll/releases/download/1.0.0/3d-camera-dll-1.0.0.zip"
diff --git a/src/yaml/memo/region-thumbnail-fix-dll.yaml b/src/yaml/memo/region-thumbnail-fix-dll.yaml
index 2f33e962..a7f2f874 100644
--- a/src/yaml/memo/region-thumbnail-fix-dll.yaml
+++ b/src/yaml/memo/region-thumbnail-fix-dll.yaml
@@ -1,9 +1,12 @@
 group: "memo"
 name: "region-thumbnail-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-region-thumbnail-fix-dll"
+  withChecksum:
+  - include: "/memo.thumbnail-fix.dll"
+    sha256: 2b17a275727012c8d9d74a84035be5bdcca480bea4b23322bab3941ed14609a5
 
 info:
   summary: "Fixes region thumbnail rendering bug"
@@ -19,6 +22,7 @@ info:
 
 ---
 assetId: "memo-region-thumbnail-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-08-17T10:12:54Z"
-url: "https://community.simtropolis.com/files/file/36396-region-thumbnail-fix-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36396-region-thumbnail-fix-dll/?do=download"
+url: "https://github.com/memo33/sc4-thumbnail-fix-dll/releases/download/1.0.0/thumbnail-fix-dll-1.0.0.zip"
diff --git a/src/yaml/memo/submenus-dll.yaml b/src/yaml/memo/submenus-dll.yaml
index b028e579..17c8cb03 100644
--- a/src/yaml/memo/submenus-dll.yaml
+++ b/src/yaml/memo/submenus-dll.yaml
@@ -1,11 +1,14 @@
 group: "memo"
 name: "submenus-dll"
-version: "1.1.4"
+version: "1.1.4-1"
 subfolder: "150-mods"
 dependencies:
 - "null-45:sc4-resource-loading-hooks"
 assets:
 - assetId: "memo-submenus-dll"
+  withChecksum:
+  - include: "/memo.submenus.dll"
+    sha256: 2d8208ac9c9ffcfa52e16f8a3e9d23758a985d298c1397706bea2fe8479bf465
 
 info:
   summary: "Adds more submenus to the game"
@@ -24,6 +27,7 @@ info:
 
 ---
 assetId: "memo-submenus-dll"
-version: "1.1.4"
+version: "1.1.4-1"
 lastModified: "2024-08-09T07:49:06Z"
-url: "https://community.simtropolis.com/files/file/36142-submenus-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36142-submenus-dll/?do=download"
+url: "https://github.com/memo33/submenus-dll/releases/download/1.1.4/submenus-dll-1.1.4.zip"
diff --git a/src/yaml/memo/transparent-texture-fix-dll.yaml b/src/yaml/memo/transparent-texture-fix-dll.yaml
index 2b3baedb..76ca8840 100644
--- a/src/yaml/memo/transparent-texture-fix-dll.yaml
+++ b/src/yaml/memo/transparent-texture-fix-dll.yaml
@@ -1,9 +1,12 @@
 group: "memo"
 name: "transparent-texture-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-transparent-texture-fix-dll"
+  withChecksum:
+  - include: "/memo.transparent-texture-fix.dll"
+    sha256: 5b472588ff9a6df1628203fe26983c00cea0bbccfc1a9f755a9a03dd9c668c17
 
 info:
   summary: "Fixes the underground/water view bug"
@@ -17,6 +20,7 @@ info:
 
 ---
 assetId: "memo-transparent-texture-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-08-08T20:09:26Z"
-url: "https://community.simtropolis.com/files/file/36379-transparent-texture-fix-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36379-transparent-texture-fix-dll/?do=download"
+url: "https://github.com/memo33/transparent-texture-fix-dll/releases/download/1.0.0/transparent-texture-fix-dll-1.0.0.zip"
diff --git a/src/yaml/neko/prop-sets.yaml b/src/yaml/neko/prop-sets.yaml
index 91acf42d..1b9f8dfd 100644
--- a/src/yaml/neko/prop-sets.yaml
+++ b/src/yaml/neko/prop-sets.yaml
@@ -13,9 +13,11 @@ info:
 
 ---
 assetId: "neko-prop-set-vol01-part1"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:39:31Z"
 url: "http://hide-inoki.com/bbs/archives/files/1482.zip"
+checksum:
+  sha256: 7fd3d6126f9fc100892c7f8d9e3e7a9b256f008f2cbfb14ee622063722c1c8b8
 
 ---
 group: "neko"
@@ -33,9 +35,11 @@ info:
 
 ---
 assetId: "neko-prop-set-vol01-part2"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:39:38Z"
 url: "http://hide-inoki.com/bbs/archives/files/1483.zip"
+checksum:
+  sha256: 3192341cf288cbc62c3e1793cab95b5e35161a6ee5504c3ffb32cc77b8686547
 
 ---
 group: "neko"
@@ -53,9 +57,11 @@ info:
 
 ---
 assetId: "neko-prop-set-vol02"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:38:58Z"
 url: "http://hide-inoki.com/bbs/archives/files/1392.zip"
+checksum:
+  sha256: af657c7443c46c143ea4d8db1c374f965f6d5887e1bc9176b5aad2f77654dada
 
 ---
 group: "neko"
@@ -73,9 +79,11 @@ info:
 
 ---
 assetId: "neko-prop-set-vol03"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:38:53Z"
 url: "http://hide-inoki.com/bbs/archives/files/1391.zip"
+checksum:
+  sha256: 0312a80ad0130bbe3718176b3e11f24cc8529e5a785eaae0d4e8efa9ad5bfc59
 
 ---
 group: "neko"
@@ -94,8 +102,10 @@ info:
 ---
 url: "http://hide-inoki.com/bbs/archives/files/1582.zip"
 assetId: "neko-prop-set-vol04-part1"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:41:09Z"
+checksum:
+  sha256: 6841c4e9887e67e9bacfb707cd0442bd30e4b5919395a1da093103d9222a1df5
 
 ---
 group: "neko"
@@ -114,8 +124,10 @@ info:
 ---
 url: "http://hide-inoki.com/bbs/archives/files/1581.zip"
 assetId: "neko-prop-set-vol04-part2"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:41:03Z"
+checksum:
+  sha256: d82617c873ca8f9afa4e9a1a673d0f4c0b2dbebe284060a25624c4ff337a582a
 
 ---
 group: "neko"
@@ -134,8 +146,10 @@ info:
 ---
 url: "http://hide-inoki.com/bbs/archives/files/1592.zip"
 assetId: "neko-prop-set-vol05"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:41:15Z"
+checksum:
+  sha256: 66fb3a6e7233a5f0f88f178ef88e90e5d8d99c9c0039c2edba3b6f2201fb996a
 
 ---
 group: "neko"
@@ -151,5 +165,7 @@ info:
 ---
 url: "http://hide-inoki.com/bbs/archives/files/1580.zip"
 assetId: "neko-texture-set-01"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:40:59Z"
+checksum:
+  sha256: 27f44a65ae0f941f97554a48175c86f81e1a69903eb9df14fc9205a4e97ee8f6
diff --git a/src/yaml/null-45/sc4-resource-loading-hooks.yaml b/src/yaml/null-45/sc4-resource-loading-hooks.yaml
index 14d35fe9..c06e422b 100644
--- a/src/yaml/null-45/sc4-resource-loading-hooks.yaml
+++ b/src/yaml/null-45/sc4-resource-loading-hooks.yaml
@@ -1,9 +1,12 @@
 group: "null-45"
 name: "sc4-resource-loading-hooks"
-version: "1.0"
+version: "1.0.1"
 subfolder: "150-mods"
 assets:
 - assetId: "null-45-sc4-resource-loading-hooks"
+  withChecksum:
+  - include: "/SC4ResourceLoadingHooks.dll"
+    sha256: 490b0c64f86837b153349367e45609a42ac135d6bc23f46d6eab807ce233078d
 
 info:
   summary: "DLL that modifies resources as the game loads them"
@@ -11,10 +14,11 @@ info:
   description: |
     This DLL plugin is a dependency for other DLLs.
   author: "Null 45"
-  website: "https://github.com/0xC0000054/sc4-resource-loading-hooks/releases"
+  website: "https://community.simtropolis.com/files/file/36242-resource-loading-hooks-dll-for-simcity-4/"
 
 ---
 assetId: "null-45-sc4-resource-loading-hooks"
-version: "1.0"
-lastModified: "2024-03-12T10:10:56Z"
-url: "https://github.com/0xC0000054/sc4-resource-loading-hooks/releases/download/v1.0/SC4ResourceLoadingHooks.zip"
+version: "1.0.1"
+lastModified: "2024-06-26T13:34:16Z"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36242-resource-loading-hooks-dll-for-simcity-4/?do=download"
+url: "https://github.com/0xC0000054/sc4-resource-loading-hooks/releases/download/v1.0.1/SC4ResourceLoadingHooks.zip"
diff --git a/src/yaml/simmaster07/extra-cheats-dll.yaml b/src/yaml/simmaster07/extra-cheats-dll.yaml
index 112826bc..66de295e 100644
--- a/src/yaml/simmaster07/extra-cheats-dll.yaml
+++ b/src/yaml/simmaster07/extra-cheats-dll.yaml
@@ -1,10 +1,16 @@
 group: "simmaster07"
 name: "extra-cheats-dll"
-version: "1.1.1-1"
+version: "1.1.1-2"
 subfolder: "150-mods"
 assets:
 - assetId: "simmaster07-extra-cheats-dll"
+  withChecksum:
+  - include:  "/ExtraExtraCheats.dll"
+    sha256: bc67b8ade3254258e9ee4410d3fb7474f2f1c5f47a9b0eefd03dd526962b759f
 - assetId: "buggi-extra-cheats-dll"
+  withChecksum:
+  - include: "/SimCity 4 Extra Cheats Plugin.dll"
+    sha256: 5ec83e5c4ae318a13bee52a77ead277ee1b776819d64261c7f483a60b1c01401
 
 info:
   summary: "Enables additional cheat codes"
diff --git a/src/yaml/simmaster07/sc4fix.yaml b/src/yaml/simmaster07/sc4fix.yaml
index 032f8c46..71dfdaf0 100644
--- a/src/yaml/simmaster07/sc4fix.yaml
+++ b/src/yaml/simmaster07/sc4fix.yaml
@@ -1,12 +1,15 @@
 group: "simmaster07"
 name: "sc4fix"
-version: "1.0.7"
+version: "1.0.7-1"
 subfolder: "150-mods"
 assets:
 - assetId: "simmaster07-sc4fix"
+  withChecksum:
+  - include: "/SC4Fix.dll"
+    sha256: 3c67a48d51212e748a535de3a4f5551ccaa0577255e1757726f7df384a828c76
 
 info:
-  summary: "Third-party patches for SC4"
+  summary: "DLL patches for SC4"
   conflicts: "Incompatible with the macOS version of the game."
   description: |
     Fixes a serialization bug that could corrupt the savegame, particularly when mods are installed that modify existing exemplars (i.e. Prop Pox).
@@ -21,6 +24,7 @@ info:
 
 ---
 assetId: "simmaster07-sc4fix"
-version: "1.0.7"
+version: "1.0.7-1"
 lastModified: "2018-01-21T05:11:27Z"
-url: "https://community.simtropolis.com/files/file/30883-sc4fix-third-party-patches-for-sc4/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/30883-sc4fix-third-party-patches-for-sc4/?do=download"
+url: "https://github.com/nsgomez/sc4fix/releases/download/rev7/SC4Fix.dll"

From ce1f0c8e30297448f9092f9e22f2ba2fe19e0e88 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Sat, 14 Dec 2024 17:22:02 +0100
Subject: [PATCH 11/13] rename `IRM.industrial-capacity` to
 `toroca:industry-quadrupler:capacity`

---
 src/yaml/t-wrecks/industrial-revolution-mod.yaml | 12 ++++++------
 src/yaml/toroca/industry-quadrupler.yaml         |  6 +++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/yaml/t-wrecks/industrial-revolution-mod.yaml b/src/yaml/t-wrecks/industrial-revolution-mod.yaml
index 283ff9e4..330236fa 100644
--- a/src/yaml/t-wrecks/industrial-revolution-mod.yaml
+++ b/src/yaml/t-wrecks/industrial-revolution-mod.yaml
@@ -28,13 +28,13 @@ dependencies:
 - "t-wrecks:maxis-prop-names-and-query-fix"
 
 variants:
-- variant: { CAM: "no", IRM.industrial-capacity: "standard" }
+- variant: { CAM: "no", toroca:industry-quadrupler:capacity: "standard" }
   assets:
   - assetId: "t-wrecks-irm-addon-set-i-d"
     include:
     - "\\.SC4Lot$"
     - "/Standard Edition/"
-- variant: { CAM: "no", IRM.industrial-capacity: "quadrupled" }
+- variant: { CAM: "no", toroca:industry-quadrupler:capacity: "quadrupled" }
   dependencies: [ "toroca:industry-quadrupler" ]
   assets:
   - assetId: "t-wrecks-irm-addon-set-i-d"
@@ -51,7 +51,7 @@ variantDescriptions:
   CAM:
     "no": "Choose this if you have not installed the Colossus Addon Mod (CAM)."
     "yes": "Choose this if you have installed the Colossus Addon Mod (CAM)."
-  IRM.industrial-capacity:
+  toroca:industry-quadrupler:capacity:
     "standard": "Maxis-default capacity"
     "quadrupled": "4x capacity, for use with the Industry Quadrupler"
 
@@ -77,13 +77,13 @@ dependencies:
 - "t-wrecks:maxis-prop-names-and-query-fix"
 
 variants:
-- variant: { CAM: "no", IRM.industrial-capacity: "standard" }
+- variant: { CAM: "no", toroca:industry-quadrupler:capacity: "standard" }
   assets:
   - assetId: "t-wrecks-irm-addon-set-i-m"
     include:
     - "\\.SC4Lot$"
     - "/Standard Edition/"
-- variant: { CAM: "no", IRM.industrial-capacity: "quadrupled" }
+- variant: { CAM: "no", toroca:industry-quadrupler:capacity: "quadrupled" }
   dependencies: [ "toroca:industry-quadrupler" ]
   assets:
   - assetId: "t-wrecks-irm-addon-set-i-m"
@@ -100,7 +100,7 @@ variantDescriptions:
   CAM:
     "no": "Choose this if you have not installed the Colossus Addon Mod (CAM)."
     "yes": "Choose this if you have installed the Colossus Addon Mod (CAM)."
-  IRM.industrial-capacity:
+  toroca:industry-quadrupler:capacity:
     "standard": "Maxis-default capacity"
     "quadrupled": "4x capacity, for use with the Industry Quadrupler"
 
diff --git a/src/yaml/toroca/industry-quadrupler.yaml b/src/yaml/toroca/industry-quadrupler.yaml
index bce44ed0..5c1328a7 100644
--- a/src/yaml/toroca/industry-quadrupler.yaml
+++ b/src/yaml/toroca/industry-quadrupler.yaml
@@ -3,14 +3,14 @@ name: "industry-quadrupler"
 version: "2.1"
 subfolder: "150-mods"
 variants:
-- variant: { IRM.industrial-capacity: "standard" }
+- variant: { toroca:industry-quadrupler:capacity: "standard" }
   # empty
-- variant: { IRM.industrial-capacity: "quadrupled" }
+- variant: { toroca:industry-quadrupler:capacity: "quadrupled" }
   assets:
   - assetId: "toroca-industry-quadrupler"
 
 variantDescriptions:
-  IRM.industrial-capacity:
+  toroca:industry-quadrupler:capacity:
     "standard": "Maxis-default capacity"
     "quadrupled": "4x capacity for Maxis industrial buildings"
 

From 2ee4f402dae437a596ef6e46b0927cd602c9fa82 Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Sun, 24 Nov 2024 09:19:04 +0100
Subject: [PATCH 12/13] update CLI documentation for channel-build, search and
 server commands

---
 docs/cli.md | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/docs/cli.md b/docs/cli.md
index 93b8fd3d..54cafe85 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -76,8 +76,18 @@ sc4pac search --threshold 20 "Pause border"    # Decrease threshold for more res
 >>> ...
 ```
 
+You can search for a URL of a STEX entry or SC4Evermore download page to find any corresponding packages:
+
+```sh
+sc4pac search "https://community.simtropolis.com/files/file/32812-save-warning/"
+>>> ...
+
+sc4pac search "https://www.sc4evermore.com/index.php/downloads/download/26-gameplay-mods/26-bsc-no-maxis"
+>>> ...
+```
+
 **Options:**
-- `--threshold <number>` Fuziness (0..100, default=50): Smaller numbers lead to more results.
+- `--threshold <number>` Fuziness (0..100, default=80): Smaller numbers lead to more results.
 
 
 ---
@@ -187,10 +197,15 @@ Build a channel locally by converting YAML files to JSON.
 **Examples:**
 ```sh
 sc4pac channel build --output "channel/json/" "channel/yaml/"
+sc4pac channel build --label Local --metadata-source-url https://github.com/memo33/sc4pac/blob/main/src/yaml/ -o channel/json channel/yaml
 ```
 
+Use the options `--label` and `--metadata-source-url` particularly for building publicly accessible channels.
+
 **Options:**
-- `-o, --output <dir>`  Output directory for JSON files
+- `-o, --output <dir>`         Output directory for JSON files
+- `--label str`                Optional short channel name for display in the UI
+- `--metadata-source-url url`  Optional base URL linking to the online YAML source files (for Edit Metadata button)
 
 
 ---
@@ -202,13 +217,18 @@ Start a local server to use the HTTP [API](api).
 
 **Example:**
 ```sh
-sc4pac server --indent 2 --profile-root profiles/profile-1/
+sc4pac server --profiles-dir profiles --indent 1
+sc4pac server --profiles-dir profiles --web-app-dir build/web                # used by GUI web
+sc4pac server --profiles-dir profiles --auto-shutdown --startup-tag [READY]  # used by GUI desktop
 ```
 
 **Options:**
-- `--port <number>`        (default: 51515)
-- `--indent <number>`      indentation of JSON responses (default: -1, no indentation)
-- `--profile-root <path>`  root directory containing `sc4pac-plugins.json` (default: current working directory), newly created if necessary; can be used for managing multiple different plugins folders
+- `--port <number>`         (default: 51515)
+- `--indent <number>`       indentation of JSON responses (default: -1, no indentation)
+- `--profiles-dir <path>`   directory containing the `sc4pac-profiles.json` file and profile sub-directories (default: current working directory), newly created if necessary
+- `--web-app-dir <path>`    optional directory containing statically served webapp files (default: no static files)
+- `--auto-shutdown`         automatically shut down the server when client closes connection to `/server.connect` (default: `--auto-shutdown=false`). This is used by the desktop GUI to ensure the port is cleared when the GUI exits.
+- `--startup-tag <string>`  optional tag to print once server has started and is listening
 
 
 ---

From 5b77d783399724c0712f014139a73bc2b5b5d74f Mon Sep 17 00:00:00 2001
From: memo <memo33@users.noreply.github.com>
Date: Sat, 14 Dec 2024 21:22:28 +0100
Subject: [PATCH 13/13] upgrade to sc4pac 0.5.0, set channel label and metadata
 source URL

---
 .github/workflows/sc4pac.yaml | 2 +-
 Makefile                      | 5 ++++-
 sc4pac-tools                  | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sc4pac.yaml b/.github/workflows/sc4pac.yaml
index dc299d22..74a6733b 100644
--- a/.github/workflows/sc4pac.yaml
+++ b/.github/workflows/sc4pac.yaml
@@ -118,7 +118,7 @@ jobs:
       - name: Build sc4pac executable
         run: cd sc4pac-tools && sbt assembly && ./sc4pac --version
       - name: Build channel and website
-        run: make gh-pages-no-lint
+        run: make LABEL=Main gh-pages-no-lint
       - name: Setup Pages
         uses: actions/configure-pages@v4
       - name: Upload artifact
diff --git a/Makefile b/Makefile
index ae9b2b6e..4043b1b2 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,9 @@
 # SC4PAC=sc4pac
 SC4PAC=./sc4pac-tools/sc4pac
 
+# LABEL=Main
+LABEL=Main-local
+
 # Rebuild all .json files, the main.js file and update the gh-pages branch.
 #
 # This assumes that you have initialized the submodule `sc4pac-tools` with:
@@ -26,7 +29,7 @@ gh-pages-no-lint:
 	cp -p ./docs/index.html ./docs/*.md ./docs/.nojekyll ./gh-pages/
 
 channel:
-	$(SC4PAC) channel build --output ./gh-pages/channel/ ./src/yaml/
+	$(SC4PAC) channel build --label $(LABEL) --metadata-source-url https://github.com/memo33/sc4pac/blob/main/src/yaml/ --output ./gh-pages/channel/ ./src/yaml/
 
 # Open e.g. http://localhost:8091/channel/?pkg=memo:essential-fixes
 host:
diff --git a/sc4pac-tools b/sc4pac-tools
index d8b1f817..aa8c16ae 160000
--- a/sc4pac-tools
+++ b/sc4pac-tools
@@ -1 +1 @@
-Subproject commit d8b1f81708548ff0bfea0c739f1550950819a803
+Subproject commit aa8c16ae789a6ff6e2fc018f384878e2ad6c66e8