🐧🧹 HealthChecks 🖳🗑️ #1708
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🐧🧹 HealthChecks 🖳🗑️ | |
| #MAX_RUNTIME: 02 Minutes */10 * * * * | |
| on: | |
| #push: | |
| workflow_dispatch: | |
| schedule: | |
| # - cron: "45 03 * * *" # 03:45 AM UTC --> 09:30 AM Morning NPT | |
| - cron: "0 */12 * * *" # Every 12 Hrs | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.TOOLPACKS }}" | |
| RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
| R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks" | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| check-post: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Debloat Runner | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/debloat.sh") | |
| continue-on-error: true | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| - name: Install Golang | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| pushd "$(mktemp -d)" >/dev/null | |
| echo "yes" | bash <(curl -qfsSL "https://git.io/go-installer") | |
| popd >/dev/null | |
| continue-on-error: true | |
| - name: Install NIX | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| "/nix/nix-installer" uninstall --no-confirm 2>/dev/null | |
| curl -qfsSL "https://install.determinate.systems/nix" | bash -s -- install linux --init none --no-confirm | |
| source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" | |
| sudo chown --recursive "runner" "/nix" | |
| echo "root ALL=(ALL:ALL) ALL" | sudo tee -a "/etc/sudoers" | |
| nix --version && nix-channel --list && nix-channel --update | |
| continue-on-error: true | |
| - name: Install Rust | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://sh.rustup.rs") -y | |
| continue-on-error: true | |
| - name: Install Zig | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| sudo rm "/usr/local/zig" -rf 2>/dev/null ; sudo rm "/usr/local/zig" -rf 2>/dev/null | |
| pushd "$(mktemp -d)" >/dev/null 2>&1 && curl -qfSLJO $(curl -qfsSL "https://ziglang.org/download/index.json" | jq -r '.master | ."x86_64-linux".tarball') | |
| find . -type f -name '*.tar*' -exec tar -xf {} \; | |
| sudo mkdir -p "/usr/local/zig" && sudo mv "$(find . -maxdepth 1 -type d | grep -v '^.$')"/* "/usr/local/zig" ; popd >/dev/null 2>&1 | |
| continue-on-error: true | |
| - name: Dos2Unix Everything | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find . -type f ! -path "./.git/*" -exec dos2unix {} \; 2>/dev/null | |
| continue-on-error: true | |
| - name: ActionLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find ".github/workflows" -type f -name "*ml" -exec actionlint {} \; | |
| continue-on-error: true | |
| - name: Shellcheck | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| find ".github" -type f -name '*.sh' -exec shellcheck --exclude="SC2261" --severity=error "{}" \; 2>/dev/null | tee "${GITHUB_WORKSPACE}/main/.github/shellcheck.txt" | |
| continue-on-error: true | |
| - name: YAMLint | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins" | |
| find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
| continue-on-error: true | |
| - name: Generate GO_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| go tool dist list --json | jq -r '["| [GOOS](https://pkg.go.dev/internal/platform) | [GOARCH](https://github.com/golang/go/blob/master/src/go/build/syslist.go) | [CGO_SUPPORT](https://web.archive.org/web/20240614051126/https://medium.com/@pengcheng1222/exploring-cgo-enabled-in-go-23cf5cf2fe88) | [NATIVE](https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63) |", "| --- | --- | --- | --- |"] + (map("| \(.GOOS) | \(.GOARCH) | \(.CgoSupported | if . then "✅" else "❌" end) | \(.FirstClass | if . then "✅" else "❌" end) |")) | join("\n")' > "$GITHUB_WORKSPACE/main/Docs/GO_TARGETS.md" | |
| continue-on-error: true | |
| - name: Generate NIX_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #https://github.com/NixOS/nixpkgs/blob/master/lib/systems/examples.nix | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| nix-instantiate --eval --strict --expr 'let pkgs = import <nixpkgs> {}; pkgsCross = pkgs.pkgsCross; in builtins.attrNames pkgsCross' | tr -d '[]"' | tr ' ' '\n' | sed '/^\s*$/d' | sed 's/^/pkgsCross./' > "$GITHUB_WORKSPACE/main/Docs/NIX_TARGETS.txt" | |
| continue-on-error: true | |
| - name: Generate RUST_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| rustup target list > "$GITHUB_WORKSPACE/main/Docs/RUST_TARGETS.txt" | |
| continue-on-error: true | |
| - name: Generate ZIG_TARGETS | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
| zig targets | jq -r '.libc[]' > "$GITHUB_WORKSPACE/main/Docs/ZIG_TARGETS.txt" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo-I ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #copy & chdir to Repo | |
| cp -r "$GITHUB_WORKSPACE/main" "$SYSTMP/REPO" && cd "$SYSTMP/REPO" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| # rm -rf "$(pwd)/aarch64_arm64_Linux" | |
| # rm -rf "$(pwd)/arm64_v8a_Android" | |
| # rm -rf "$(pwd)/x86_64_Linux" | |
| # rm -rf "$(pwd)/x64_Windows" | |
| #Upload to Pub | |
| wait ; echo | |
| echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
| rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| ##Scripts_aarch64_Linux | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_aarch64_Linux" & | |
| ##Scripts_arm64_v8a_Android | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_arm64_v8a_Android" & | |
| ##Scripts_x86_64_Linux | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_x86_64_Linux" & | |
| ##Scripts_x64_Windows | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_x64_Windows" & | |
| ##Clean Pretty Formatted | |
| wait ; echo | |
| echo "[" $(cat metadata.json.tmp_aarch64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_arm64_v8a_Android | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_arm64_v8a_Android" & | |
| echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_x64_Windows | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x64_Windows" & | |
| ##Update Metadata | |
| wait ; echo | |
| rclone copyto --checksum "./metadata.json_aarch64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/metadata.json" & | |
| rclone copyto --checksum "./metadata.json_arm64_v8a_Android" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/metadata.json" & | |
| rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/metadata.json" & | |
| rclone copyto --checksum "./metadata.json_x64_Windows" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/metadata.json" & | |
| ##Update Repo | |
| wait ; echo | |
| cat "./metadata.json_aarch64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/metadata.json" | |
| cat "./metadata.json_arm64_v8a_Android" > "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins/metadata.json" | |
| cat "./metadata.json_x86_64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/metadata.json" | |
| cat "./metadata.json_x64_Windows" > "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins/metadata.json" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #-----------------------------------------------------# | |
| #-----------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/arm64_v8a_Android/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to $GITHUB_WORKSPACE/main/arm64_v8a_Android | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/FILE.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/SIZE.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
| cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.txt" >> "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
| echo '```' >> "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SHA256SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/METADATA.json" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" 2>/dev/null | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| #-----------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/aarch64_arm64_Linux/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to $GITHUB_WORKSPACE/main/aarch64_arm64_Linux | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/FILE.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.min.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.min.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.toml" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.toml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.yaml" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.yaml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SIZE.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
| cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
| echo '```' >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SHA256SUM.txt" 2>/dev/null | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| #-----------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/x86_64_Linux/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to $GITHUB_WORKSPACE/main/x86_64_Linux | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/FILE.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.min.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.min.json" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.toml" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.toml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.yaml" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.yaml" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/SIZE.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
| cat "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
| echo '```' >> "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/SHA256SUM.txt" 2>/dev/null | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| #-----------------------------------------------------# | |
| - name: Parse ("bin.ajam.dev/x64_Windows/") Bins | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Fetch from r2 | |
| #Copy to $GITHUB_WORKSPACE/main/x64_Windows | |
| rclone copyto --checksum "r2:/bin/x64_Windows/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/x64_Windows/BLAKE3SUM.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x64_Windows/FILE.txt" "$GITHUB_WORKSPACE/main/x64_Windows/FILE.txt" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x64_Windows/SIZE.txt" "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.txt" 2>/dev/null | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
| cat "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.txt" >> "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
| echo '```' >> "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
| rclone copyto --checksum "r2:/bin/x64_Windows/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/x64_Windows/SHA256SUM.txt" 2>/dev/null | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| - name: Generate Repo Metadata (git-sizer) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Dust sizes | |
| echo '```mathematica' > "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee -a "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee "$GITHUB_WORKSPACE/main/.github/SIZE.txt" | |
| echo '```' >> "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
| continue-on-error: true | |
| - name: Parse Build Logs (aarch64_arm64_Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #--------------------# | |
| ##aarch64_arm64_Linux## | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/BUILD.BIN.log.txt" -o "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| #Secrets & Tokens | |
| sed -i '/.*github_pat.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*ghp_.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*access_key_id.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*secret_access_key.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*cloudflarestorage.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
| echo -e '```mathematica' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Broken GH Repos | |
| echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -B5 -A3 -in ".* Authentication failed .*" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Eget Fails | |
| echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -B5 -in "asset .* not found" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Failed URLs | |
| echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Failed Installs | |
| echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -in "command not found" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Dynamically Linked | |
| echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -in "dynamically linked" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Not correct binary | |
| echo -e "\n\n[-] Incorrect Arch --> # Not for aarch64\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -ivn "aarch64\|text" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Build Failures | |
| echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| grep -in "No such file or directory" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #Empty Bins | |
| echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| #END | |
| echo -e '```' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
| continue-on-error: true | |
| - name: Parse Build Logs (x86_64_Linux) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #--------------------# | |
| ##x86_64_Linux## | |
| #Secrets & Tokens | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/BUILD.BIN.log.txt" -o "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*github_pat.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*ghp_.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*access_key_id.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*secret_access_key.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| sed -i '/.*cloudflarestorage.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
| echo -e '```mathematica' > "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Broken GH Repos | |
| echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -B5 -A3 -in ".* Authentication failed .*" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Eget Fails | |
| echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -B5 -in "asset .* not found" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Failed URLs | |
| echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Failed Installs | |
| echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -in "command not found" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Dynamically Linked | |
| echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -in "dynamically linked" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Not correct binary | |
| echo -e "\n\n[-] Incorrect Arch --> # Not for x86_64\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -ivn "x86-64\|text" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Broken Builds | |
| echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| grep -in "No such file or directory" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #Empty Bins | |
| echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| #END | |
| echo -e '```' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
| continue-on-error: true | |
| - name: Generate Markdown (Main README.md) | |
| run: | | |
| #Presets | |
| set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Generate $GITHUB_WORKSPACE/main/README.md | |
| cat "$GITHUB_WORKSPACE/main/INFO.md" > "$GITHUB_WORKSPACE/main/README.md" | |
| ##bin.ajam.dev/arm64_v8a_Android/ | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "arm64-v8a-Android") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "arm64-v8a-Android") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_arm64_v8a_Android/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| ##bin.ajam.dev/aarch64_arm64_Linux | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_aarch64_arm64_Linux/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| ##bin.ajam.dev/x86_64_Linux/ | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_x86_64_Linux/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| ##bin.ajam.dev/x64_Windows/ | |
| unset MAIN_N BASE_N TOTAL_PACKAGES | |
| MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "AMD64-Windows_NT") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
| BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "AMD64-Windows_NT") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
| export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_x64_Windows/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| continue-on-error: true | |
| - name: Scrape Public Issues | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #https://docs.github.com/en/rest/issues/issues | |
| export DATE_CUTOFF="2024-00-00T00:00:00Z" | |
| TEMP_ISSUES="$(mktemp -d)" && export TEMP_ISSUES="${TEMP_ISSUES}" | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo -e "\n[+] Using Temp Dir : ${TEMP_ISSUES}" | |
| #Fetch: `static binary` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: Static Binary ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| #Fetch: `static build` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: Static Build ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+build+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| wait ; echo | |
| #Fetch: `static compile` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: Static Compile ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+compile+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| #Fetch: `musl binary` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: MUSL Binary ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| wait ; echo | |
| #Fetch: `musl build` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: MUSL Build ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+build+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| #Fetch: `musl compile` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: MUSL Compile ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+compile+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| wait ; echo | |
| #Fetch: `release binary` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: Release Binary ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=release+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| #Fetch: `release static` | |
| (for page in {1..10}; do | |
| echo -e "\n[+] GH API: Release Binary ($page/10)\n" | |
| curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=release+static+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
| sleep 2 | |
| done) & | |
| wait ; echo | |
| #Sort | |
| find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --unique --reverse -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
| #find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --numeric-sort --reverse --unique -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
| #find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --version-sort --reverse --unique -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
| continue-on-error: true | |
| #- name: Ensure Healthy Commit Nums (RESET ON >= 1000) | |
| # run: | | |
| # #Presets | |
| # set +x ; set +e | |
| # #--------------# | |
| # pushd "$(mktemp -d)" > /dev/null 2>&1 && git clone --filter="blob:none" "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" | |
| # cd "$(find . -maxdepth 1 -type d -exec basename {} \; | grep -Ev '^\.$' | xargs -I {} realpath {})" | |
| # CLONED_DIR="$(realpath .)" && export CLONED_DIR="$CLONED_DIR" | |
| # COMMIT_NUMS="$(git rev-list --count HEAD)" && export COMMIT_NUMS="$COMMIT_NUMS" | |
| # popd > /dev/null 2>&1 ; rm -rf "$CLONED_DIR" 2>/dev/null | |
| # cd "$GITHUB_WORKSPACE/main" | |
| # if [ "$COMMIT_NUMS" -gt 1000 ]; then | |
| # echo -e "\n[+] Total number of commits exceeds 1000. ($COMMIT_NUMS)\n" | |
| # cd "$GITHUB_WORKSPACE/main" | |
| # git config "user.name" "Azathothas" | |
| # git config "user.email" "[email protected]" | |
| # git checkout --orphan temp | |
| # git add --all --verbose && git commit -m "Purge (Re:Init)" | |
| # git branch -D "main" | |
| # git branch -m "main" | |
| # git push --set-upstream origin main --force | |
| # echo "RESET_HISTORY=YES" >> "$GITHUB_ENV" | |
| # else | |
| # echo -e "\n[+] Total number of commits looks healthy. ($COMMIT_NUMS)\n" | |
| # echo "RESET_HISTORY=NO" >> "$GITHUB_ENV" | |
| # fi | |
| # pushd "$GITHUB_WORKSPACE/main" >/dev/null 2>&1 && git pull origin main && popd >/dev/null 2>&1 | |
| # continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| # Date Time | |
| NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
| echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| # - name: Git Pull | |
| # run: | | |
| # cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
| # continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
| commit_user_email: [email protected] # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
| commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
| #push_options: '--force' | |
| continue-on-error: true | |
| #-----------------------------------------------------# | |
| - name: rClone Generate Metadata (MAIN) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| #--------------------# | |
| ##arm64_v8a_Android## | |
| rclone lsjson --fast-list "r2:/bin/arm64_v8a_Android/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/arm64_v8a_Android/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_arm64_v8a_Android" & | |
| ##aarch64_arm64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_aarch64_arm64_Linux" & | |
| ##x86_64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/x86_64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_x86_64_Linux" & | |
| ##x64_Windows## | |
| rclone lsjson --fast-list "r2:/bin/x64_Windows/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x64_Windows/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_x64_Windows" & | |
| ##Clean Pretty Formatted | |
| wait ; echo | |
| echo "[" $(cat metadata.json.tmp_arm64_v8a_Android | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_arm64_v8a_Android" & | |
| echo "[" $(cat metadata.json.tmp_aarch64_arm64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_arm64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
| echo "[" $(cat metadata.json.tmp_x64_Windows | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x64_Windows" & | |
| ##Update Metadata | |
| rclone copyto --checksum "./metadata.json_arm64_v8a_Android" "r2:/bin/arm64_v8a_Android/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| rclone copyto --checksum "./metadata.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/bin/x86_64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| rclone copyto --checksum "./metadata.json_x64_Windows" "r2:/bin/x64_Windows/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
| wait ; echo | |
| continue-on-error: true | |
| - name: rClone Generate Metadata (BASEUTILS) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| #--------------------# | |
| ##arm64_v8a_Android## | |
| rclone copy "r2:/bin/arm64_v8a_Android/Baseutils/" "./metadata_Baseutils_arm64_v8a_Android_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##aarch64_arm64_Linux## | |
| rclone copy "r2:/bin/aarch64_arm64_Linux/Baseutils/" "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##x86_64_Linux## | |
| rclone copy "r2:/bin/x86_64_Linux/Baseutils/" "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##x64_Windows## | |
| rclone copy "r2:/bin/x64_Windows/Baseutils/" "./metadata_Baseutils_x64_Windows_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
| ##Merge (INFO) | |
| wait ; echo | |
| find "./metadata_Baseutils_arm64_v8a_Android_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_arm64_v8a_Android_INFO_ALL.json" & | |
| ##aarch64_arm64_Linux## | |
| find "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" & | |
| find "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" & | |
| find "./metadata_Baseutils_x64_Windows_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_x64_Windows_INFO_ALL.json" & | |
| ##Enrich | |
| wait ; echo | |
| rclone lsjson --fast-list "r2:/bin/arm64_v8a_Android/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/arm64_v8a_Android/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_arm64_v8a_Android" & | |
| ##aarch64_arm64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
| ##x86_64_Linux## | |
| rclone lsjson --fast-list "r2:/bin/x86_64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
| ##x64_Windows## | |
| rclone lsjson --fast-list "r2:/bin/x64_Windows/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x64_Windows/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_x64_Windows" & | |
| ##Clean | |
| wait ; echo | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_arm64_v8a_Android" & | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
| sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_x64_Windows" & | |
| ##Merge (ALL) | |
| wait ; echo | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_arm64_v8a_Android" "./metadata_Baseutils_arm64_v8a_Android_INFO_ALL.json" > "./metadata_Baseutils.json_arm64_v8a_Android" & | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_aarch64_arm64_Linux" & | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_x86_64_Linux" "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_x86_64_Linux" & | |
| jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_x64_Windows" "./metadata_Baseutils_x64_Windows_INFO_ALL.json" > "./metadata_Baseutils.json_x64_Windows" & | |
| ##Update | |
| rclone copyto --checksum "./metadata_Baseutils.json_arm64_v8a_Android" "r2:/bin/arm64_v8a_Android/Baseutils/METADATA.json" & | |
| rclone copyto --checksum "./metadata_Baseutils.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/Baseutils/METADATA.json" & | |
| rclone copyto --checksum "./metadata_Baseutils.json_x86_64_Linux" "r2:/bin/x86_64_Linux/Baseutils/METADATA.json" & | |
| rclone copyto --checksum "./metadata_Baseutils.json_x64_Windows" "r2:/bin/x64_Windows/Baseutils/METADATA.json" & | |
| wait ; echo | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo-II ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #touch "$HOME/.rclone.conf" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
| #chdir to Repo | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| #Upload to Pub | |
| echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
| rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-arm64v8a-Android: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/arm64_v8a_Android) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_arm64_v8a_Android" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/arm64_v8a_Android/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/arm64_v8a_Android_FILE" | |
| rclone copyto "${SYSTMP}/arm64_v8a_Android_FILE" "r2:/bin/arm64_v8a_Android/FILE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/arm64_v8a_Android_SIZE.txt" | |
| rclone copyto "${SYSTMP}/arm64_v8a_Android_SIZE.txt" "r2:/bin/arm64_v8a_Android/SIZE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/arm64_v8a_Android_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/arm64_v8a_Android_BLAKE3SUM" "r2:/bin/arm64_v8a_Android/BLAKE3SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/arm64_v8a_Android_SHA256SUM" | |
| rclone copyto "${SYSTMP}/arm64_v8a_Android_SHA256SUM" "r2:/bin/arm64_v8a_Android/SHA256SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/arm64_v8a_Android) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-arm64-Linux: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_aarch64" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/aarch64_arm64_Linux/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/aarch64_Linux_FILE" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_FILE" "r2:/bin/aarch64_arm64_Linux/FILE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/aarch64_Linux_SIZE.txt" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_SIZE.txt" "r2:/bin/aarch64_arm64_Linux/SIZE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_BLAKE3SUM" "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_SHA256SUM" | |
| rclone copyto "${SYSTMP}/aarch64_Linux_SHA256SUM" "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-x86-64-Linux: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/x86_64_Linux) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_x86_64" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/x86_64_Linux/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/x86_64_Linux_FILE" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_FILE" "r2:/bin/x86_64_Linux/FILE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/x86_64_Linux_SIZE.txt" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_SIZE.txt" "r2:/bin/x86_64_Linux/SIZE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_BLAKE3SUM" "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_SHA256SUM" | |
| rclone copyto "${SYSTMP}/x86_64_Linux_SHA256SUM" "r2:/bin/x86_64_Linux/SHA256SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/x86_64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| enrich-x64-Windows: | |
| runs-on: ubuntu-latest | |
| needs: [check-post] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
| sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
| sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
| continue-on-error: true | |
| - name: rClone Check Metadata (bin.ajam.dev/x64_Windows) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| BINDIR="${SYSTMP}/toolpack_x86_64" && export BINDIR="${BINDIR}" | |
| mkdir -p "${BINDIR}" | |
| rclone mount "r2:/bin/x64_Windows/" "${BINDIR}" \ | |
| --daemon \ | |
| --allow-other \ | |
| --no-modtime \ | |
| --read-only \ | |
| --attr-timeout 15s \ | |
| --checkers 100 \ | |
| --buffer-size 512M \ | |
| --dir-cache-time 5m \ | |
| --low-level-retries 10 \ | |
| --multi-thread-streams 16 \ | |
| --poll-interval 15s \ | |
| --transfers 16 \ | |
| --vfs-read-wait 0 \ | |
| --vfs-cache-mode full \ | |
| --vfs-cache-max-size 20G \ | |
| --vfs-cache-poll-interval 15s \ | |
| --vfs-read-chunk-size 128M \ | |
| --vfs-read-chunk-size-limit off \ | |
| --user-agent="${USER_AGENT}" ; df -h | |
| #File | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/x64_Windows_FILE" | |
| rclone copyto "${SYSTMP}/x64_Windows_FILE" "r2:/bin/x64_Windows/FILE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Size (Dust) | |
| dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/x64_Windows_SIZE.txt" | |
| rclone copyto "${SYSTMP}/x64_Windows_SIZE.txt" "r2:/bin/x64_Windows/SIZE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #BLAKE3SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/x64_Windows_BLAKE3SUM" | |
| rclone copyto "${SYSTMP}/x64_Windows_BLAKE3SUM" "r2:/bin/x64_Windows/BLAKE3SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #SHA256SUM | |
| cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/x64_Windows_SHA256SUM" | |
| rclone copyto "${SYSTMP}/x64_Windows_SHA256SUM" "r2:/bin/x64_Windows/SHA256SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Purge | |
| #rm -rf "${BINDIR}" | |
| sudo fusermount3 -uz "${BINDIR}" ; df -h | |
| continue-on-error: true | |
| - name: rClone Enrich Metadata (bin.ajam.dev/x64_Windows) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Run | |
| chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
| dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
| bash "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
| cd "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| merge-all: | |
| runs-on: ubuntu-latest | |
| needs: [enrich-arm64v8a-Android, enrich-arm64-Linux, enrich-x86-64-Linux, enrich-x64-Windows] | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Setup Env | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| - name: Merge & Update ("bin.ajam.dev/METADATA.json") | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| ##Fetch | |
| #arm64_v8a_Android | |
| curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/arm64_v8a_Android.json" | |
| #aarch64_arm64_Linux | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| #cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
| #x86_64_Linux | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| #cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
| #x64_Windows | |
| curl -qfsSL "https://bin.ajam.dev/x64_Windows/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
| curl -qfsSL "https://bin.ajam.dev/x64_Windows/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
| jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x64_Windows.json" | |
| ##Merge | |
| if jq empty "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" 2>/dev/null; then | |
| #jq -s '[{ "host": "aarch64-Linux" } + .[0][]], [{ "host": "x86_64-Linux" } + .[1][]], [{ "host": "arm64-v8a-Android" } + .[2][]], [{ "host": "AMD64-Windows_NT" } + .[3][]]' "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" | jq 'sort_by(.host)' > "$SYSTMP/METADATA.json" | |
| jq -s '[{ "host": "aarch64-Linux" } + .[0][], { "host": "x86_64-Linux" } + .[1][], { "host": "arm64-v8a-Android" } + .[2][], { "host": "AMD64-Windows_NT" } + .[3][]]' "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" | jq 'sort_by(.host)' > "$SYSTMP/METADATA.json" | |
| if jq empty "$SYSTMP/METADATA.json" 2>/dev/null; then | |
| ##rClone Update | |
| echo -e "\n Updating Metadata.json\n" | |
| rclone copyto --checksum "$SYSTMP/METADATA.json" "r2:/bin/METADATA.json" --check-first --checkers 2000 --transfers 1000 | |
| ##Copy to REPO | |
| rsync -av --checksum "$SYSTMP/METADATA.json" "$GITHUB_WORKSPACE/main/METADATA.json" | |
| #YAML | |
| cat "$GITHUB_WORKSPACE/main/METADATA.json" | yj -jy > "$GITHUB_WORKSPACE/main/METADATA.yaml" | |
| fi | |
| fi | |
| continue-on-error: true | |
| - name: Git Pull | |
| run: | | |
| cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
| continue-on-error: true | |
| - name: Update Repo | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| #arm64_v8a_Android | |
| rclone copyto --checksum "r2:/bin/arm64_v8a_Android/METADATA.json" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/METADATA.json" -o "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | |
| #aarch64_arm64_Linux | |
| rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | |
| #x86_64_Linux | |
| rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| #curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | |
| #x64_Windows | |
| rclone copyto --checksum "r2:/bin/x64_Windows/METADATA.json" "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
| continue-on-error: true | |
| - name: Generate README (aarch64) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.yaml" | |
| #Gen README | |
| cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/README.md" | |
| sed 's/||//g' -i "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Generate README (arm64-v8a) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.yaml" | |
| #Gen README | |
| cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/README.md" | |
| sed 's/||//g' -i "$GITHUB_WORKSPACE/main/arm64_v8a_Android/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Generate README (x86_64) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.yaml" | |
| #Gen README | |
| cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/x86_64_Linux/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/x86_64_Linux/README.md" | |
| sed 's/||//g' -i "$GITHUB_WORKSPACE/main/x86_64_Linux/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Generate README (X64-Windows) | |
| run: | | |
| #presets | |
| set +x ; set +e | |
| #-------------# | |
| if jq empty "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" 2>/dev/null; then | |
| #Convert to YAML | |
| cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.yaml" | |
| #Gen README | |
| cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/x64_Windows/DETAILED.md" | |
| { echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/x64_Windows/README.md" | |
| sed 's/||//g' -i "$GITHUB_WORKSPACE/main/x64_Windows/README.md" | |
| fi | |
| continue-on-error: true | |
| - name: Check Health (arm64_v8a_Android) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/arm64_v8a_Android/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Check Health (aarch64_arm64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/aarch64_arm64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Check Health (x86_64_Linux) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/x86_64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/x86_64_Linux/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "$GITHUB_WORKSPACE/main/x86_64_Linux/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Check Health (x64_Windows) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| rclone copyto "r2:/bin/x64_Windows/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| #Duplicates | |
| jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/x64_Windows/DUPLICATES.txt" | |
| #Duplicates (overwrites) | |
| yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
| readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
| for dupe in "${dupes[@]}"; do | |
| names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | paste -sd ' ') | |
| echo "$dupe --> $names_and_paths" | |
| done | tee "$GITHUB_WORKSPACE/main/x64_Windows/DUPLICATES_OVERWRITES.txt" | |
| #Dates | |
| jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/x64_Windows/BUILD_DATES.txt" | |
| continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| # Date Time | |
| NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
| echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
| commit_user_email: [email protected] # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
| commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
| #push_options: '--force' | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# |