lib/fetch-{libc++,newlib}: exit on error

.github/workflows/ci.yml

@@ -22,11 +22,18 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: false # LVGL makefile manually installs the submodule
+
       - uses: carlosperate/arm-none-eabi-gcc-action@v1
         with:
           release: '10.3-2021.07' # The arm-none-eabi-gcc release to use.
+
       - name: setup-riscv-toolchain
         run: sudo apt-get install -y gcc-riscv64-unknown-elf
+
+      - name: Disable wget progress output
+        run: |
+          echo "verbose = off" >> $HOME/.wgetrc
+
       - name: ci-format
         run: pushd examples; ./format_all.sh || exit; popd
 
@@ -42,12 +49,20 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: recursive
+
       - uses: carlosperate/arm-none-eabi-gcc-action@v1
         with:
           release: '10.3-2021.07' # The arm-none-eabi-gcc release to use.
+
       - name: setup-riscv-toolchain
         run: sudo apt-get install -y gcc-riscv64-unknown-elf
+
+      - name: Disable wget progress output
+        run: |
+          echo "verbose = off" >> $HOME/.wgetrc
+
       - name: ci-build
         run: pushd examples; ./build_all.sh || exit; popd
+
       - name: ci-debug-build
         run: pushd examples/blink; make debug RAM_START=0x20004000 FLASH_INIT=0x30051 || exit; popd

lib/fetch-libc++.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# Exit on error:
+set -e
+
 GCC_VERSION=$1
 
 if [ $GCC_VERSION = "14.1.0" ]; then
@@ -27,28 +30,54 @@ else
   CHECK_SHA_CMD="sha256sum -c"
 fi
 
-let FOUND=0
-
-# Try from each mirror until we successfully download a .zip file.
-for MIRROR in ${MIRRORS[@]}; do
-  URL=$MIRROR/$ZIP_FILE
-  echo "Fetching libc++ from ${MIRROR}..."
-  echo "  Fetching ${URL}..."
-  # Note: There must be two space characters for `shasum` (sha256sum doesn't care)
-  wget -O $ZIP_FILE  "$URL" && (echo "$GCC_SHA  $ZIP_FILE" | $CHECK_SHA_CMD)
-  if [ $? -ne 0 ]; then
-    echo "  WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
-  else
-    let FOUND=1
-    break
-  fi
-done
+FOUND=0
 
-if [[ $FOUND -ne 0 ]]; then
-  echo "Unpacking $ZIP_FILE..."
-  unzip -q $ZIP_FILE
-  echo "Done upacking $ZIP_FILE..."
-else
-  echo "ERROR: Unable to find tock-libc++"
-  exit -1
-fi
+# We must ensure that multiple parallel fetch and unzip operations
+# don't trample over each other, which we do by obtaining a write-lock
+# on the ZIP file that's being downloaded / extracted.
+#
+# This will also truncate any already downloaded ZIP file, which is
+# fine because we'll overwrite it anyways.
+NONBLOCK_LOCK_ACQ_FAIL=0
+while true; do
+    : >> ${ZIP_FILE}
+    {
+	if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
+            flock -n $fd || NONBLOCK_LOCK_ACQ_FAIL=1
+	    if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
+		# Try again, blocking this time:
+		echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
+		continue
+	    fi
+	else
+	    flock $fd
+	fi
+        echo "Acquired lock on file ${ZIP_FILE}" >&2
+
+        # Try from each mirror until we successfully download a .zip file.
+        for MIRROR in ${MIRRORS[@]}; do
+          URL=$MIRROR/$ZIP_FILE
+          echo "Fetching libc++ from ${MIRROR}..."
+          echo "  Fetching ${URL}..."
+          # Note: There must be two space characters for `shasum` (sha256sum doesn't care)
+          wget -O $ZIP_FILE "$URL" && (echo "$GCC_SHA  $ZIP_FILE" | $CHECK_SHA_CMD)
+          if [ $? -ne 0 ]; then
+            echo "  WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
+          else
+            FOUND=1
+            break
+          fi
+        done
+
+        if [[ $FOUND -ne 0 ]]; then
+          echo "Unpacking $ZIP_FILE..."
+          # -n: don't overwrite existing files, -q: quiet mode
+          unzip -n -q $ZIP_FILE
+          echo "Done upacking $ZIP_FILE..."
+	  exit 0
+        else
+          echo "ERROR: Unable to find tock-libc++"
+          exit -1
+        fi
+    } {fd}<${ZIP_FILE}
+done

lib/fetch-newlib.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# Exit on error:
+set -e
+
 NEWLIB_VERSION=$1
 
 if [ $NEWLIB_VERSION = "4.4.0.20231231" ]; then
@@ -25,28 +28,54 @@ else
   CHECK_SHA_CMD="sha256sum -c"
 fi
 
-let FOUND=0
-
-# Try from each mirror until we successfully download a .zip file.
-for MIRROR in ${MIRRORS[@]}; do
-  URL=$MIRROR/$ZIP_FILE
-  echo "Fetching newlib from ${MIRROR}..."
-  echo "  Fetching ${URL}..."
-  # Note: There must be two space characters for `shasum` (sha256sum doesn't care)
-  wget -O $ZIP_FILE "$URL" && (echo "$NEWLIB_SHA  $ZIP_FILE" | $CHECK_SHA_CMD)
-  if [ $? -ne 0 ]; then
-    echo "  WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
-  else
-    let FOUND=1
-    break
-  fi
-done
+FOUND=0
 
-if [[ $FOUND -ne 0 ]]; then
-  echo "Unpacking $ZIP_FILE..."
-  unzip -q $ZIP_FILE
-  echo "Done upacking $ZIP_FILE..."
-else
-  echo "ERROR: Unable to find tock-newlib"
-  exit -1
-fi
+# We must ensure that multiple parallel fetch and unzip operations
+# don't trample over each other, which we do by obtaining a write-lock
+# on the ZIP file that's being downloaded / extracted.
+#
+# This will also truncate any already downloaded ZIP file, which is
+# fine because we'll overwrite it anyways.
+NONBLOCK_LOCK_ACQ_FAIL=0
+while true; do
+    : >> ${ZIP_FILE}
+    {
+	if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
+            flock -n $fd || NONBLOCK_LOCK_ACQ_FAIL=1
+	    if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
+		# Try again, blocking this time:
+		echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
+		continue
+	    fi
+	else
+	    flock $fd
+	fi
+        echo "Acquired lock on file ${ZIP_FILE}" >&2
+
+        # Try from each mirror until we successfully download a .zip file.
+        for MIRROR in ${MIRRORS[@]}; do
+          URL=$MIRROR/$ZIP_FILE
+          echo "Fetching newlib from ${MIRROR}..."
+          echo "  Fetching ${URL}..."
+          # Note: There must be two space characters for `shasum` (sha256sum doesn't care)
+          wget -O $ZIP_FILE "$URL" && (echo "$NEWLIB_SHA  $ZIP_FILE" | $CHECK_SHA_CMD)
+          if [ $? -ne 0 ]; then
+            echo "  WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
+          else
+            FOUND=1
+            break
+          fi
+        done
+
+        if [[ $FOUND -ne 0 ]]; then
+          echo "Unpacking $ZIP_FILE..."
+          # -n: don't overwrite existing files, -q: quiet mode
+          unzip -n -q $ZIP_FILE
+          echo "Done upacking $ZIP_FILE..."
+	  exit 0
+        else
+          echo "ERROR: Unable to find tock-newlib"
+          exit -1
+        fi
+    } {fd}<${ZIP_FILE}
+done