build: INFENG-885: add lock-api-state.sh pre-commit check

.github/workflows/pre-commit.yml

@@ -23,6 +23,11 @@ jobs:
         with:
           python-version: '3.9'
           cache: 'pip'
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.0'
+      - name: "Install protobuf dependencies."
+        run: "make get-deps-proto"
       # this action is not getting new features, but isn't deprecated yet
       # switch to just running pre-commit locally when/if that happens that's
       # why this one specifies a full version; presumably there will be a

.pre-commit-config.yaml

@@ -72,6 +72,20 @@ repos:
         description: "Runs `golangci-lint`, install https://github.com/golangci/golangci-lint"
         exclude: ^(proto/pkg|hpc-ard-launcher-go/launcher/)
         require_serial: true
+      - id: lock-api-state
+        name: "Generate buf image (lock-api-state.sh)"
+        entry: ./pre-commit/lock-api-state.sh
+        language: "script"
+        description: "Fails if the existing protobuf definitions would generate a new buf.image.bin."
+        files: '^proto/'
+        pass_filenames: false
+      - id: lock-published-urls
+        name: "Generate all_published_urls_ever.json (lock-published-urls.sh)"
+        entry: ./pre-commit/lock-published-urls.sh
+        language: "script"
+        description: "Ensure all redirect URLs are generated at commit time."
+        files: '^docs/'
+        pass_filenames: false
 
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.28.0

docs/Makefile

@@ -114,6 +114,10 @@ check: attributions.rst
 	git ls-files -z '*.rst' | xargs -0 rstfmt -w $(RSTFMT_WIDTH) --check
 	python3 redirects.py check
 
+.PHONY: lock-published-urls
+lock-published-urls: attributions.rst
+	python3 redirects.py publish
+
 .PHONY: get-deps
 get-deps:
 	pip install -r requirements.txt

pre-commit/lock-api-state.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Regenerate proto/buf.image.bin
+make -C proto gen-buf-image
+
+# If proto/buf.image.bin has been modified locally, then we have changes to
+# commit, and the status check returns a 1 and fails. Otherwise, it returns a 0
+# and succeeds.
+git diff --exit-code -- proto/buf.image.bin
+exit $?

pre-commit/lock-published-urls.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+make -C docs lock-published-urls
+
+# If docs/.redirects/all_published_urls_ever.json has been modified locally,
+# then `redirects.py publish` has made modifications that we need to commit.
+git diff --exit-code -- docs/.redirects/all_published_urls_ever.json
+exit $?

tools/scripts/README.md

@@ -105,10 +105,3 @@ The relevant metadata are:
 * `Webui`: boolean indicating if this is a dependency of the WebUI
 
 For instructions on how to use `gen-attributions.py`, just run it with no arguments.
-
-## `lock-api-state.sh`: Lock-in current API state
-
-We use [buf.build](https://docs.buf.build/) to provide backward compatibility check
-for API changes in Determined. By running this script on each release, we store a
-snapshot of the API state and on each following change `buf` compares the new state
-with the old to ensure that backward compatibility is preserved.