From 4c60693a000dbc25fe4cd429c908508dcb6b2ffd Mon Sep 17 00:00:00 2001 From: a-gave Date: Sun, 17 Nov 2024 11:49:06 +0100 Subject: [PATCH] ci: add multi branch build lime-system: add apk option in 92_add-lime-repos ci: update feeds scheme to build C++ packages for 3 lastest openwrt branches the new scheme make these packages available at https://github.com/libremesh/lime-feed referenced by apk in /etc/apk/repositories as @libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_main/x86_64/packages.adb ci: disable sign packages in main openwrt branch not yet supported, there is an active pull Support sign apk feed https://github.com/openwrt/gh-action-sdk/pull/46 to install packages use "apk add --allow-untrusted lime-docs" --- .github/workflows/build.yml | 32 +++- .github/workflows/multi-arch-build.yml | 141 ++++++++++++------ README.md | 36 +++-- .../files/etc/uci-defaults/92_add-lime-repos | 49 ++++-- 4 files changed, 186 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff9d82db2..017bc8712 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,22 +12,38 @@ on: - ".github/workflows/*" - "libremesh.mk" - "packages/**" + workflow_dispatch: jobs: +# Build all packages for x86_64 +# Use openwrt branches 'main' and 'openwrt-24.10' to produce two feeds: +# one for 'apk' and one for the older 'opkg'. +# NOTE: this doesn't sign packages for apk build: - name: Build ${{ github.ref }} + name: Build ${{ matrix.version }} ${{ github.ref }} runs-on: ubuntu-latest - + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - { version: main, output_path: 'apk/'} + - { version: openwrt-24.10, output_path: ''} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build packages - uses: openwrt/gh-action-sdk@v5 + uses: openwrt/gh-action-sdk@v7 env: - ARCH: "x86_64" + ARCH: "x86_64-${{ matrix.version }}" FEEDNAME: "libremesh" IGNORE_ERRORS: "n m y" - KEY_BUILD: "${{ secrets.KEY_BUILD }}" + KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}" + INDEX: 1 + NO_DEFAULT_FEEDS: 1 + NO_REFRESH_CHECK: 1 + NO_SHFMT_CHECK: 1 + # V: sc - name: Set package destination run: | @@ -36,12 +52,12 @@ jobs: echo "DEST_DIR=$TAG" >> $GITHUB_ENV - name: Upload packages to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} external_repository: libremesh/lime-feed publish_dir: bin/packages/x86_64/libremesh/ - destination_dir: ${{ env.DEST_DIR }} + destination_dir: ${{ matrix.output_path }}${{ env.DEST_DIR }} # - name: Upload packages to S3 # uses: jakejarvis/s3-sync-action@master diff --git a/.github/workflows/multi-arch-build.yml b/.github/workflows/multi-arch-build.yml index 241027325..62627ea41 100644 --- a/.github/workflows/multi-arch-build.yml +++ b/.github/workflows/multi-arch-build.yml @@ -14,74 +14,129 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +# Build packages using the openwrt sdk +# for each architecture supported +# in openwrt branchs 'main', 'stable', 'old-stable' +# +# NOTE: +# This list of branches will be updated manually +# - at next openwrt release, ideally adding '25.xx' and removing '23.05' +# - whether it will be identified an incompatibility of the current libremesh code with an openwrt branch +env: + BUILD_BRANCHES: "main 24.10 23.05" + +# workaround: 'make package/shared-state-async/check' fails with 'Package HASH check failed' + PACKAGES: "shared-state-bat_hosts" + jobs: + generate_matrix: + name: 'Generate matrix' + runs-on: ubuntu-latest + outputs: + matrix_builds: ${{ steps.define_matrix.outputs.matrix_builds }} + steps: + - name: Define matrix of branchs and archs + id: define_matrix + run: | + JSON='[' + FIRST_BUILD=1 + echo "${{ env.BUILD_BRANCHES }}" + for version in ${{ env.BUILD_BRANCHES }}; do + + VERSION=$([ "$version" == "main" ] \ + && echo "main" || echo "openwrt-$version") + echo $VERSION + + VERSION_PATH=$([ "$version" == "main" ] \ + && echo "snapshots/packages" || echo "releases/packages-$version") + echo $VERSION_PATH + + OUTPUT_PATH=$([ "$version" == "main" ] \ + && echo "openwrt_main" || echo "openwrt_$version") + echo $OUTPUT_PATH + + wget -r -nH --no-parent --level 1 --accept html -P. \ + --directory-prefix=. https://downloads.openwrt.org/$VERSION_PATH/ + + for arch in $(ls ./$VERSION_PATH/ | grep -v html); do + [[ $FIRST_BUILD -ne 1 ]] && JSON="$JSON"',' + FIRST_BUILD=0 + + JSON="$JSON"'{"version": "'"$VERSION"'" ,"arch": "'"$arch"'", "output_path": "'"$OUTPUT_PATH"'" }' + echo $JSON + done + done + + matrix_include='{"include": '"$JSON"']}' + echo "matrix_builds=${matrix_include}" >> "$GITHUB_OUTPUT" + build: - name: build ${{ matrix.arch }} ${{ github.ref }} + name: build ${{ matrix.arch }}-${{ matrix.version }} runs-on: ubuntu-latest + needs: generate_matrix strategy: fail-fast: false - max-parallel: 1 - matrix: - arch: - - aarch64_cortex-a53 - - aarch64_cortex-a72 - - aarch64_generic - - arm_arm1176jzf-s_vfp - - arm_arm926ej-s - - arm_cortex-a15_neon-vfpv4 - - arm_cortex-a5_vfpv4 - - arm_cortex-a7 - - arm_cortex-a7_neon-vfpv4 - - arm_cortex-a7_vfpv4 - - arm_cortex-a8_vfpv3 - - arm_cortex-a9 - - arm_cortex-a9_neon - - arm_cortex-a9_vfpv3-d16 - - arm_fa526 - - arm_mpcore - - arm_xscale - - i386_pentium-mmx - - i386_pentium4 - - mips64_octeonplus - - mips_24kc - - mips_4kec - - mips_mips32 - - mipsel_24kc - - mipsel_24kc_24kf - - mipsel_74kc - - mipsel_mips32 - - powerpc_464fp - - powerpc_8548 - - riscv64_riscv64 - - x86_64 - + max-parallel: 10 + matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }} + # matrix: + # version: ['main'] + # arch: ['x86_64'] + # output_path: ['openwrt_main'] steps: - uses: actions/checkout@v4 - - name: Build packages ${{ matrix.arch }} + - name: Build packages ${{ matrix.arch }}-${{ matrix.version }} uses: openwrt/gh-action-sdk@v7 env: - ARCH: "${{ matrix.arch }}" + ARCH: "${{ matrix.arch }}-${{ matrix.version }}" FEEDNAME: "libremesh" IGNORE_ERRORS: "n m y" - KEY_BUILD: "${{ secrets.KEY_BUILD }}" - PACKAGES: "shared-state-bat_hosts" + KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}" + PACKAGES: "${{ env.PACKAGES }}" INDEX: 1 NO_DEFAULT_FEEDS: 1 NO_REFRESH_CHECK: 1 NO_SHFMT_CHECK: 1 + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.arch }}-${{ matrix.version }} + path: bin/packages/${{ matrix.arch }}/libremesh/ + + publish: + name: publish ${{ github.ref }} + runs-on: ubuntu-latest + if: ${{ always() }} + needs: build + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }} + # matrix: + # version: ['main'] + # arch: ['x86_64'] + # output_path: ['openwrt_main'] + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: "${{ matrix.arch }}-${{ matrix.version }}" + path: artifacts + - name: Set package destination run: | + ls ./artifacts export TAG=$(echo "${{ github.ref }}" | cut -d '/' -f 3- | perl -pe 's/v([0-9])/$1/') echo "$TAG" echo "DEST_DIR=$TAG" >> $GITHUB_ENV - name: Upload packages to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} external_repository: libremesh/lime-feed - publish_dir: bin/packages/${{ matrix.arch }}/libremesh/ - destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.arch }} + publish_dir: ./artifacts/ + destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.output_path }}/${{ matrix.arch }}/ diff --git a/README.md b/README.md index 29ea7d419..b76cb4410 100644 --- a/README.md +++ b/README.md @@ -56,22 +56,41 @@ Start an ImageBuilder of your choice, for example ath79-generic if your device i ```shell mkdir ./images/ -docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.3 +docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.5 ``` If your device is not part of ath79-generic profiles, you can replace it with another <target>-<subtarget> combination. For knowing which target and subtarget is best suited for your router, check out the page about it in the [OpenWrt's Table of Hardware][OpenWrt-ToH]. Within the container, add the `lime-packages` feeds: +**opkg on openwrt 24.10.x and previous** + ```shell echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf -echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf echo "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b ``` +To use the new `shared-state-async` add this repo: +- replace `openwrt_23.05` with the openwrt branch +- replace `mips_24kc` with the architecture of your device based on target/subtarget + +```shell +echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_23.05/mips_24kc" >> repositories.conf +``` -If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected <target>/<subtarget>. +**apk on openwrt main branch** +``` +echo "@libremesh https://feed.libremesh.org/apk/master/packages.adb" >> repositories +echo "@libremesh_profiles https://feed.libremesh.org/apk/profiles/packages.adb" >> repositories +``` +To use the new `shared-state-async` add this repo: +- replace `openwrt_main` with the openwrt branch +- replace `mips_24kc` with the architecture of your device based on target/subtarget + +```shell +echo "@libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_main/mips_24kc/packages.adb" >> repositories +``` Ideally add your own `lime-community` files within the container in the folder `./files/etc/config/`. To find possible options consult the @@ -94,14 +113,9 @@ Your images should be available outside of the container in the `./images/` fold Go to . Find your device. Click on the folder symbol right after "Links: ". Alternatively, find your device in [OpenWrt's Table of Hardware][OpenWrt-ToH], find the image download link, remove the filename from the right side of the link and put the result in your browsers address bar. Scroll down and download openwrt-imagebuilder-*. Unpack the file and open a terminal inside the directory. Add the `lime-packages` feed: -```shell -echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf -echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf -echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf -echo "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b -echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b -``` -If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected <target>/<subtarget>. +Follow the same steps at With Docker +- **opkg on openwrt 24.10.x and previous** +- **apk on openwrt main branch** Create an image with ```shell diff --git a/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos b/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos index 59a05fc28..1065e8c86 100755 --- a/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos +++ b/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos @@ -5,21 +5,43 @@ exit 0 } -feeds_file="/etc/opkg/limefeeds.conf" +repo=$([ -f /etc/opkg/distfeed.conf ] && echo "opkg" \ + || ([ -f /etc/apk/repositories.d/distfeeds.list ] && echo "apk" ) ) || { + echo "Package manager not found - skipping" + exit 0 + } -[ -f "$feeds_file" ] && { - echo "LibreMesh opkg feeds already defined - skipping" - exit 0 +feeds_file=$([ $repo == 'opkg' ] && echo "/etc/opkg/limefeeds.conf" \ + || ([ $repo == 'apk' ] && echo "/etc/apk/repositories.d/limefeeds.list") ) + +dist_feeds_file=$([ $repo == 'opkg' ] && echo "/etc/opkg/distfeeds.conf" \ + || ([ $repo == 'apk' ] && echo "/etc/apk/repositories.d/distfeeds.list") ) + +$( ([ $repo == 'opkg' ] && [ -f "$feeds_file" ]) \ + || ([ $repo == 'apk' ] && $(grep -q '@libremesh' $feeds_file)) ) && +{ + echo "LibreMesh $repo feeds already defined - skipping" + exit 0; +} + +arch="$(grep OPENWRT_ARCH /etc/os-release | sed 's/OPENWRT_ARCH=\"\(.*\)\"/\1/')" + +openwrt_branch_ref="$(grep -m 1 "openwrt.org/" $dist_feeds_file | sed 's|.*openwrt.org/\(.*\)|\1|' )" +openwrt_branch=$(echo $openwrt_branch_ref | grep -q 'snapshots' && echo 'openwrt_main' ) +$(echo $openwrt_branch_ref | grep -q 'releases') && { + branch_n="$(echo $openwrt_branch_ref | sed 's/releases\///')" + openwrt_branch="openwrt_${branch_n:0:5}" } -arch_packages="$(grep OPENWRT_ARCH /etc/os-release | sed 's/OPENWRT_ARCH=\"\(.*\)\"/\1/')" +main_feed_path=$([ $repo == 'opkg' ] && echo "" \ + || ([ $repo == 'apk' ] && echo "apk/") ) [ "$LIME_CODENAME" == "development" ] && { - packages_url="http://feed.libremesh.org/master"; - arch_packages_url="http://feed.libremesh.org/arch_packages/master/$arch_packages"; + packages_url="http://feed.libremesh.org/${main_feed_path}master"; + arch_packages_url="http://feed.libremesh.org/arch_packages/master/$openwrt_branch/$arch/"; } || { - packages_url="http://feed.libremesh.org/$LIME_RELEASE" - arch_packages_url="http://feed.libremesh.org/arch_packages/$LIME_RELEASE/$arch_packages"; + packages_url="http://feed.libremesh.org/${main_feed_path}$LIME_RELEASE" + arch_packages_url="http://feed.libremesh.org/arch_packages/$LIME_RELEASE/$openwrt_branch/$arch/"; } profiles_url="http://feed.libremesh.org/profiles" @@ -27,9 +49,16 @@ profiles_url="http://feed.libremesh.org/profiles" key_name="a71b3c8285abd28b" key_content="RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" -echo "Configuring official LibreMesh opkg feeds" +echo "Configuring official LibreMesh $repo feeds" +[ $repo == 'opkg' ] && { echo "src/gz libremesh $packages_url" > "$feeds_file" echo "src/gz libremesh_arch_packages $arch_packages_url" >> "$feeds_file" echo "src/gz profiles $profiles_url" >> "$feeds_file" echo "untrusted comment: signed by libremesh.org key $key_name" > "/etc/opkg/keys/$key_name" echo "$key_content" >> "/etc/opkg/keys/$key_name" +} +[ $repo == 'apk' ] && { +echo "@libremesh $packages_url/packages.adb" >> "$feeds_file" +echo "@libremesh_arch_packages $arch_packages_url/packages.adb" >> "$feeds_file" +echo "@profiles $profiles_url/packages.adb" >> "$feeds_file" +} \ No newline at end of file