Merge pull request #56 from memo33/lint-with-checksums

Lint with checksums, upgrade to sc4pac 0.5.0

.github/sc4e-check-updates.py

@@ -56,7 +56,7 @@ def main() -> int:
             continue
 
         # check URLs
-        url = doc.get('url')
+        url = doc.get('nonPersistentUrl') or doc.get('url')
         if url is None:
             continue  # not an asset
         m = url_id_pattern.fullmatch(url)

.github/st-check-updates.py

@@ -65,7 +65,7 @@ def main() -> int:
                 continue
 
             # find all STEX file IDs
-            url = doc.get('url')
+            url = doc.get('nonPersistentUrl') or doc.get('url')
             if url is None:
                 continue  # not an asset
             m = url_id_pattern.fullmatch(url)
@@ -98,7 +98,7 @@ def main() -> int:
             continue
 
         # check URLs
-        url = doc.get('url')
+        url = doc.get('nonPersistentUrl') or doc.get('url')
         if url is None:
             continue  # not an asset
         m = url_id_pattern.fullmatch(url)

.github/workflows/sc4pac.yaml

@@ -118,7 +118,7 @@ jobs:
       - name: Build sc4pac executable
         run: cd sc4pac-tools && sbt assembly && ./sc4pac --version
       - name: Build channel and website
-        run: make gh-pages-no-lint
+        run: make LABEL=Main gh-pages-no-lint
       - name: Setup Pages
         uses: actions/configure-pages@v4
       - name: Upload artifact

Makefile

@@ -5,6 +5,9 @@
 # SC4PAC=sc4pac
 SC4PAC=./sc4pac-tools/sc4pac
 
+# LABEL=Main
+LABEL=Main-local
+
 # Rebuild all .json files, the main.js file and update the gh-pages branch.
 #
 # This assumes that you have initialized the submodule `sc4pac-tools` with:
@@ -26,7 +29,7 @@ gh-pages-no-lint:
 	cp -p ./docs/index.html ./docs/*.md ./docs/.nojekyll ./gh-pages/
 
 channel:
-	$(SC4PAC) channel build --output ./gh-pages/channel/ ./src/yaml/
+	$(SC4PAC) channel build --label $(LABEL) --metadata-source-url https://github.com/memo33/sc4pac/blob/main/src/yaml/ --output ./gh-pages/channel/ ./src/yaml/
 
 # Open e.g. http://localhost:8091/channel/?pkg=memo:essential-fixes
 host:

docs/_sidebar.md

@@ -2,7 +2,7 @@
   * [Getting started](/)
   * [CLI](cli.md)
   * [API](api.md)
-  * [Adding metadata](metadata.md)
+  * [Metadata format](metadata.md)
   * [About](about.md)
 - Packages
   * [Highlights](packages.md)

docs/cli.md

@@ -76,8 +76,18 @@ sc4pac search --threshold 20 "Pause border"    # Decrease threshold for more res
 >>> ...
 ```
 
+You can search for a URL of a STEX entry or SC4Evermore download page to find any corresponding packages:
+
+```sh
+sc4pac search "https://community.simtropolis.com/files/file/32812-save-warning/"
+>>> ...
+
+sc4pac search "https://www.sc4evermore.com/index.php/downloads/download/26-gameplay-mods/26-bsc-no-maxis"
+>>> ...
+```
+
 **Options:**
-- `--threshold <number>` Fuziness (0..100, default=50): Smaller numbers lead to more results.
+- `--threshold <number>` Fuziness (0..100, default=80): Smaller numbers lead to more results.
 
 
 ---
@@ -187,10 +197,15 @@ Build a channel locally by converting YAML files to JSON.
 **Examples:**
 ```sh
 sc4pac channel build --output "channel/json/" "channel/yaml/"
+sc4pac channel build --label Local --metadata-source-url https://github.com/memo33/sc4pac/blob/main/src/yaml/ -o channel/json channel/yaml
 ```
 
+Use the options `--label` and `--metadata-source-url` particularly for building publicly accessible channels.
+
 **Options:**
-- `-o, --output <dir>`  Output directory for JSON files
+- `-o, --output <dir>`         Output directory for JSON files
+- `--label str`                Optional short channel name for display in the UI
+- `--metadata-source-url url`  Optional base URL linking to the online YAML source files (for Edit Metadata button)
 
 
 ---
@@ -202,13 +217,18 @@ Start a local server to use the HTTP [API](api).
 
 **Example:**
 ```sh
-sc4pac server --indent 2 --profile-root profiles/profile-1/
+sc4pac server --profiles-dir profiles --indent 1
+sc4pac server --profiles-dir profiles --web-app-dir build/web                # used by GUI web
+sc4pac server --profiles-dir profiles --auto-shutdown --startup-tag [READY]  # used by GUI desktop
 ```
 
 **Options:**
-- `--port <number>`        (default: 51515)
-- `--indent <number>`      indentation of JSON responses (default: -1, no indentation)
-- `--profile-root <path>`  root directory containing `sc4pac-plugins.json` (default: current working directory), newly created if necessary; can be used for managing multiple different plugins folders
+- `--port <number>`         (default: 51515)
+- `--indent <number>`       indentation of JSON responses (default: -1, no indentation)
+- `--profiles-dir <path>`   directory containing the `sc4pac-profiles.json` file and profile sub-directories (default: current working directory), newly created if necessary
+- `--web-app-dir <path>`    optional directory containing statically served webapp files (default: no static files)
+- `--auto-shutdown`         automatically shut down the server when client closes connection to `/server.connect` (default: `--auto-shutdown=false`). This is used by the desktop GUI to ensure the port is cleared when the GUI exits.
+- `--startup-tag <string>`  optional tag to print once server has started and is listening
 
 
 ---

docs/metadata.md

@@ -1,4 +1,4 @@
-# Adding metadata
+# Metadata format
 
 This page details how to write, for an existing mod, custom metadata that is understood by *sc4pac*.
 The metadata is stored in [YAML](https://en.wikipedia.org/wiki/YAML) files which can be edited in any text editor
@@ -83,6 +83,31 @@ On SC4Evermore, grab the *Changed* timestamp from the info box on the download p
 For other sites, use the available info on the download page or, when supported by the server,
 use the `Last-Modified` HTTP header of the download URL. Shorthand for cURL users: `curl -I -L '<url>'`.
 
+### `checksum`
+
+An optional sha256 checksum can be added for verifying file integrity.
+If present, the checksum of the file is checked directly after download, before extracting the archive.
+```yaml
+checksum:
+  sha256: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+```
+This is recommended for files that are downloaded using http instead of https.
+
+### `nonPersistentUrl`
+
+When the `url` points to a specific _persistent_ version of a file,
+a `nonPersistentUrl` can be added that always points to the _latest_ version of the file.
+```yaml
+nonPersistentUrl: "https://community.simtropolis.com/files/file/25137-hogwarts-castle/?do=download"
+url: "https://github.com/....../v950/hogwarts-castle.zip"
+```
+
+?> Mainly, this is useful for DLLs released on GitHub and Simtropolis.
+   The `url` would point to the current release on GitHub, while the `nonPersistentUrl` links to Simtropolis.
+   The `nonPersistentUrl` is never used for downloading, but can be used by tools to check for updates.
+   As the file is downloaded from a specific release on GitHub,
+   this avoids intermittent checksum errors when the metadata has not yet been updated after a new release has been uploaded to Simtropolis.
+
 ### `archiveType`
 
 This is only needed for assets containing Clickteam exe-installers. It is not needed for NSIS exe-installers.
@@ -236,12 +261,32 @@ Details:
 - The matching is case-insensitive for file-system independence.
 - In any case, always both `include` and `exclude` filters are evaluated.
   If one or both are absent, they default to the following behavior:
-  - If the `include` filter is absent or empty, then by default all files with file type .dat/.sc4model/.sc4lot/.sc4desc/.sc4/.dll are included.
-  - If the `exclude` filter is absent or empty, then by default all file types other than .dat/.sc4model/.sc4lot/.sc4desc/.sc4/.dll are excluded.
+  - If the `include` filter is absent or empty, then by default all files with file type .dat/.sc4model/.sc4lot/.sc4desc/.sc4 are included.
+  - If the `exclude` filter is absent or empty, then by default all file types other than .dat/.sc4model/.sc4lot/.sc4desc/.sc4 are excluded.
+- All extracted files without checksum must be DBPF files.
 
 ?> If you anticipate file names changing with future updates of the original upload,
    consider using regular expressions to make the matching more generic, so that the `include` filter keeps working after the updates.
 
+### `withChecksum`
+
+In addition to the `include`/`exclude` fields, you can use a `withChecksum` block to only include files if their checksum matches.
+This is _required_ for DLL files (code mods) and other non-DBPF file types.
+The checksums are verified after the files are extracted from an asset,
+but before they are moved from a temporary location into the plugins folder loaded by the game.
+```yaml
+assets:
+- assetId: "dumbledore-hogwarts-castle"
+  include:
+  - "/Hogwarts/"  # only DBPF files
+  withChecksum:
+  - include: "/magic.dll"
+    sha256: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+```
+In contrast to the `checksum` field of an asset, this is the sha256 hash of the extracted file itself (e.g. hash of the .dll instead of the .zip file).
+
+?> When using `withChecksum`, it is recommended to also add a [`nonPersistentUrl`](#nonPersistentUrl) to the corresponding asset definition.
+
 ### `info`
 
 Additional descriptive information.

src/yaml/aqua877/jrs-prop-pack.yaml

@@ -11,6 +11,8 @@ info:
 
 ---
 assetId: "aqua-877-jrs-prop-pack-signs"
-version: "1"
+version: "1-1"
 lastModified: "2020-08-26T12:37:13Z"
 url: "http://hide-inoki.com/bbs/archives/files/1107.zip"
+checksum:
+  sha256: 28cbea30c01afe556884236210d0557c68787acc84c93535b9878aba10635ee3

src/yaml/jenx/fauna.yaml

@@ -2,7 +2,7 @@
 assetId: "jenx-fauna"
 version: "1.0"
 lastModified: "2023-09-15T16:33:15-07:00"
-url: "https://www.sc4evermore.com/index.php/downloads?task=download.send&id=155:sc4d-lex-legacy-jenx-fauna&catid=25"
+url: "https://www.sc4evermore.com/index.php/downloads?task=download.send&id=155:sc4d-lex-legacy-jenx-fauna"
 
 ---
 group: "jenx"

src/yaml/jim/carprop-pack.yaml

@@ -13,6 +13,8 @@ assets:
 
 ---
 assetId: hide-inoki-jim-carprop
-version: "1.2"
+version: "1.2-1"
 lastModified: "2012-11-11T23:17:00Z"
 url: http://hide-inoki.com/bbs/archives/files/1598.zip
+checksum:
+  sha256: d1a4e2735436867e46b847a83266874fe4963517d88f147dc795bdb9b4862d1c

src/yaml/memo/3d-camera-dll.yaml

@@ -1,9 +1,12 @@
 group: "memo"
 name: "3d-camera-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-3d-camera-dll"
+  withChecksum:
+  - include: "/memo.3dcamera.dll"
+    sha256: 281406de45ea19ebd886b2584d2417781cca6ea874973cc4bd4b1f2e9d5ee216
 
 info:
   summary: "Allows setting arbitrary camera angles"
@@ -29,6 +32,7 @@ info:
 
 ---
 assetId: "memo-3d-camera-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-04-01T08:26:25Z"
-url: "https://community.simtropolis.com/files/file/36188-3d-camera-dll-for-simcity-4/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36188-3d-camera-dll-for-simcity-4/?do=download"
+url: "https://github.com/memo33/sc4-3d-camera-dll/releases/download/1.0.0/3d-camera-dll-1.0.0.zip"

src/yaml/memo/region-thumbnail-fix-dll.yaml

@@ -1,9 +1,12 @@
 group: "memo"
 name: "region-thumbnail-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-region-thumbnail-fix-dll"
+  withChecksum:
+  - include: "/memo.thumbnail-fix.dll"
+    sha256: 2b17a275727012c8d9d74a84035be5bdcca480bea4b23322bab3941ed14609a5
 
 info:
   summary: "Fixes region thumbnail rendering bug"
@@ -19,6 +22,7 @@ info:
 
 ---
 assetId: "memo-region-thumbnail-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-08-17T10:12:54Z"
-url: "https://community.simtropolis.com/files/file/36396-region-thumbnail-fix-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36396-region-thumbnail-fix-dll/?do=download"
+url: "https://github.com/memo33/sc4-thumbnail-fix-dll/releases/download/1.0.0/thumbnail-fix-dll-1.0.0.zip"

src/yaml/memo/submenus-dll.yaml

@@ -1,11 +1,14 @@
 group: "memo"
 name: "submenus-dll"
-version: "1.1.4"
+version: "1.1.4-1"
 subfolder: "150-mods"
 dependencies:
 - "null-45:sc4-resource-loading-hooks"
 assets:
 - assetId: "memo-submenus-dll"
+  withChecksum:
+  - include: "/memo.submenus.dll"
+    sha256: 2d8208ac9c9ffcfa52e16f8a3e9d23758a985d298c1397706bea2fe8479bf465
 
 info:
   summary: "Adds more submenus to the game"
@@ -24,6 +27,7 @@ info:
 
 ---
 assetId: "memo-submenus-dll"
-version: "1.1.4"
+version: "1.1.4-1"
 lastModified: "2024-08-09T07:49:06Z"
-url: "https://community.simtropolis.com/files/file/36142-submenus-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36142-submenus-dll/?do=download"
+url: "https://github.com/memo33/submenus-dll/releases/download/1.1.4/submenus-dll-1.1.4.zip"

src/yaml/memo/transparent-texture-fix-dll.yaml

@@ -1,9 +1,12 @@
 group: "memo"
 name: "transparent-texture-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 subfolder: "150-mods"
 assets:
 - assetId: "memo-transparent-texture-fix-dll"
+  withChecksum:
+  - include: "/memo.transparent-texture-fix.dll"
+    sha256: 5b472588ff9a6df1628203fe26983c00cea0bbccfc1a9f755a9a03dd9c668c17
 
 info:
   summary: "Fixes the underground/water view bug"
@@ -17,6 +20,7 @@ info:
 
 ---
 assetId: "memo-transparent-texture-fix-dll"
-version: "1.0.0"
+version: "1.0.0-1"
 lastModified: "2024-08-08T20:09:26Z"
-url: "https://community.simtropolis.com/files/file/36379-transparent-texture-fix-dll/?do=download"
+nonPersistentUrl: "https://community.simtropolis.com/files/file/36379-transparent-texture-fix-dll/?do=download"
+url: "https://github.com/memo33/transparent-texture-fix-dll/releases/download/1.0.0/transparent-texture-fix-dll-1.0.0.zip"