Skip to content

Update repo

Update repo #48574

Workflow file for this run

name: Update repo
on:
push:
branches:
- master
paths:
- '.github/workflows/repo-update.yml'
schedule:
- cron: '0,30 0-23 * * *'
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: true
jobs:
check-repo:
runs-on: ubuntu-latest
strategy:
matrix:
target_repo: [main@aarch64, main@arm, main@i686, main@x86_64, x11@aarch64, x11@arm, x11@i686, x11@x86_64, root@aarch64, root@arm, root@i686, root@x86_64]
fail-fast: false
steps:
- name: Clone check-tools
run: git clone "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/check-tools"
- name: Import key of Termux-Pacman
run: gpg --keyserver keyserver.ubuntu.com --recv-keys 998de27318e867ea976ba877389ceed64573dfca
- name: Clone termux-packages
run: |
cd check-tools
bash ./clone_repos.sh
- name: Running a scan
run: |
target_repo='${{ matrix.target_repo }}'
REPO="${target_repo%%@*}"
ARCH="${target_repo#*@}"
TOTAL="RESULT.${REPO}.${ARCH}"
DIR_REPO="termux-packages-repo/"
case "$REPO" in
"main") DIR_REPO+="packages";;
"x11") DIR_REPO+="x11-packages";;
"root") DIR_REPO+="root-packages";;
*)
echo "Error: repo '${REPO}' is unknown"
exit 0;;
esac
cd check-tools
if [ ! -d "$DIR_REPO" ]; then
echo "Error: '$DIR_REPO' not found."
exit 0
fi
bash ./check_pkgs_by_repo.sh "$REPO" "$ARCH" "$DIR_REPO"
# Removing packages...
DELETED_FILE="./output/$(date +%s)-deleted_termux-${REPO}_packages.txt"
for reason in NOTSUPPORTARCH UNKNOWN; do
infofile="./output/${reason}.${REPO}.${ARCH}"
if [ -f "${infofile}" ]; then
cat "${infofile}" >> "${DELETED_FILE}"
fi
done
if [ -f "${DELETED_FILE}" ]; then
# Install aws-cli
git clone https://github.com/termux-pacman/aws-cli-action
./aws-cli-action/setup.sh '${{ secrets.AWS_ACCESS_KEY_ID }}' '${{ secrets.AWS_ACCESS_KEY }}' '${{ secrets.AWS_REGION }}'
# Import gpg key
mkdir gk
echo "${{ secrets.SF_GPG_BOT }}" > gk/termux-pacman-bot.gpg
gpg --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --import gk/termux-pacman-bot.gpg
rm -fr gk
# Upload file
echo "$(sed '/^[[:space:]]*$/d' ${DELETED_FILE} | sort | uniq)" > "${DELETED_FILE}"
gpg --batch --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --detach-sign --use-agent -u '${{ secrets.KEY_GPG_BOT }}' --no-armor "${DELETED_FILE}"
for format_file in "" ".sig"; do
aws s3 cp "${DELETED_FILE}${format_file}" s3://'${{ secrets.SFPU }}'/${REPO}/${ARCH}/"${DELETED_FILE##*/}${format_file}"
done
rm "${DELETED_FILE}.sig"
fi
mkdir total
touch ./total/${TOTAL}
for RESULT in "./output/"{NOTFOUND,DOESNOTMATCH}".${REPO}.${ARCH}"; do
if [ -f ${RESULT} ]; then
cat ${RESULT} | awk '{printf $1 "\n"}' >> ./total/${TOTAL}
fi
done
tar cf total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz ./total
- name: Store RESULT.* files
if: always()
uses: actions/upload-artifact@v4
with:
name: total-${{ matrix.target_repo }}-${{ github.sha }}
path: ./check-tools/total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz
if-no-files-found: error
update-repo:
needs: check-repo
runs-on: ubuntu-latest
steps:
- name: Clone repository
run: git clone https://github.com/termux-pacman/termux-packages.git termux-packages-pacman
- name: Get RESULT.* files
uses: actions/[email protected]
with:
path: ./
- name: Creating a REBUILD-LIST-BUILD
run: |
for i in ./total-*/total-*.tar.gz; do
tar xf $i -C ./
done
rm -fr ./total-*/
touch ./REBUILD-LIST-BUILD
cd termux-packages-pacman
for i in $(sort ../total/RESULT.* | uniq); do
BUILD_SOURCE=$(ls */${i}/build.sh 2> /dev/null) || true
if [ -z "${BUILD_SOURCE}" ]; then
BUILD_SOURCE=$(ls */*/${i}.subpackage.sh 2> /dev/null) || true
if [ -n "${BUILD_SOURCE}" ]; then
BUILD_SOURCE="$(dirname ${BUILD_SOURCE})/build.sh"
else
echo "Source package '${i}' was not found, skip"
continue
fi
fi
if ! $(grep -q '^'${BUILD_SOURCE}'$' ../REBUILD-LIST-BUILD) && [ $(($(expr $(expr $(date +%s) - $(git log -n 1 --date=format:'%s' --format=%cd -- $(dirname ${BUILD_SOURCE}))) / 3600) > 8)) = 1 ]; then
echo "${BUILD_SOURCE}" >> ../REBUILD-LIST-BUILD
fi
done
cd ..
- name: Clone root repository
run: git clone https://github.com/termux/termux-packages.git
- name: Update repo
run: |
info() {
echo "==> $1"
}
commit() {
echo "-> $1"
}
install_list_issues() {
if [ -f ../LIST-ISSUES ]; then
rm ../LIST-ISSUES
fi
curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -s -X GET -G \
"https://api.github.com/repos/${{ github.repository }}/issues" -o ../LIST-ISSUES
}
# Edit
info "Edit"
cd termux-packages-pacman
rm -fr */
cd ../termux-packages
cp -r * ../termux-packages-pacman
# Get list pkg
info "Get list pkg"
cd ../termux-packages-pacman
list_files=" "
for i in $(git status -s packages root-packages x11-packages | awk '{print $2}'); do
dir_sp=(${i//// })
if ! grep -q " ${dir_sp[0]}/${dir_sp[1]} " <<< "$list_files"; then
list_files+="${dir_sp[0]}/${dir_sp[1]} "
source_build="${dir_sp[0]}/${dir_sp[1]}/build.sh"
if $(grep -q '^'${source_build}'$' ../REBUILD-LIST-BUILD); then
sed -i "/^${source_build////'\/'}$/d" ../REBUILD-LIST-BUILD
fi
fi
done
list_files=($list_files)
# Sort list pkg
info "Sort list pkg"
cd ../termux-packages
declare -A list_values
list_sort=""
list_sha=""
for i in ${list_files[@]}; do
sha_file=$(git log -n 1 --pretty=format:%H -- $i)
value="$(git rev-list --count $sha_file)"
list_values[${value}]=$((${list_values[${value}]}+1))
list_sort+="${value}-${list_values[${value}]} "
list_sha+="${sha_file} "
done
list_sort=($list_sort)
list_sha=($list_sha)
copy_list_sort=($(sort -n <(printf "%s\n" "${list_sort[@]}")))
search_index() {
for i in "${!list_sort[@]}"; do
if [[ "${list_sort[$i]}" = "${1}" ]]; then
echo "${i}";
fi
done
}
sort_list_files=""
for i in ${copy_list_sort[@]}; do
index=$(search_index "$i")
if [[ "${i#*-}" = "1" ]]; then
sort_list_files+=" ${list_sha[$index]}###"
else
sort_list_files+="&&"
fi
sort_list_files+="${list_files[$index]}"
done
# Start upload
info "Start upload"
cd ../termux-packages-pacman
needbuild=false
if $(git log -1 --pretty=%B | grep -q '%needbuild'); then
needbuild=true
fi
git config --global user.name "termux-pacman-bot"
git config --global user.email "[email protected]"
git remote set-url origin "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/termux-packages.git"
commit "Update system repo"
git add .
git reset packages root-packages x11-packages README.md SECURITY.md repo.json
{
git commit -m "Update repo"
git push origin master
} || true
commit "Update repo packages"
install_list_issues
for i in $sort_list_files; do
i_sp=(${i//###/ })
commit " ${i_sp[1]}"
cd ../termux-packages
commit_file="$(git show -s --format=%B ${i_sp[0]})"
cd ../termux-packages-pacman
git add $(sed 's/&&/ /g' <<< "${i_sp[1]}")
{
if $needbuild; then
git commit -m "$(sed 's/%ci:no-build//g; s/\[skip ci\]//g' <<< $commit_file)"
else
git commit -m "$commit_file"
fi
git push origin master
} || true
for j in $(sed 's/&&/ /g' <<< "${i_sp[1]}"); do
while read LINE; do
USER=$(awk -F ' // ' '{printf $1}' <<< "$LINE")
STATE=$(awk -F ' // ' '{printf $2}' <<< "$LINE")
TITLE=$(awk -F ' // ' '{printf $4}' <<< "$LINE")
if [ $STATE != "open" ] || [ $USER != "termux-pacman-bot" ] || [[ $TITLE != "[AUTO-CHECK]: failed to recompile ${j}" ]]; then
continue
fi
NUMBER=$(awk -F ' // ' '{printf $3}' <<< "$LINE")
GH_TOKEN="${{ secrets.TOKEN }}" gh issue close ${NUMBER} \
-c "The package has been updated - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${j})."
install_list_issues
break
done <<< $(jq -r '.[] | "\(.user.login) \("//") \(.state) \("//") \(.number) \("//") \(.title)"' ../LIST-ISSUES)
done
done
# Recompile packages if necessary
if [ -n "$(cat ../REBUILD-LIST-BUILD)" ]; then
info "Rebuild packages"
listchanged=""
for i in $(cat ../REBUILD-LIST-BUILD); do
# It is necessary to create an issue because the package has
# a compilation error even after recompilation.
if [[ $(git log -n 1 --format=%B -- "${i}") = "rebuild($(dirname ${i})): auto check by @termux-pacman-bot" ]] && \
[[ $(git log -n 1 --format=%an -- "${i}") = "termux-pacman-bot" ]] && \
[[ $(git log -n 1 --format=%ae -- "${i}") = "[email protected]" ]]; then
if ! $(jq -r '.[] | "\(.user.login) \("/") \(.state) \("/") \(.title)"' ../LIST-ISSUES | grep -q "^termux-pacman-bot / open / \[AUTO-CHECK\]:.*$(dirname ${i})$"); then
commit "Create an issue on $(dirname ${i})"
GH_TOKEN="${{ secrets.TOKEN }}" gh issue create \
--title "[AUTO-CHECK]: failed to recompile $(dirname ${i})" \
--body "$(echo -e "This issue was automatically generated because the '$(dirname ${i})' package was not recompiled successfully.\n\nCommit by recompilation - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${i})")" \
--label "auto recompilation failed" \
--assignee Maxython
fi
continue
fi
echo "" >> ./${i}
git add ./${i}
listchanged+=" ${i}"
done
if [ -n "${listchanged}" ]; then
git commit -m "$(echo -e "termux-pacman/auto-check-repo: preparing to recompile packages\n[skip ci]")"
git push origin master
for i in ${listchanged}; do
sed -i '$d' ./${i}
git add ./${i}
git commit -m "rebuild($(dirname ${i})): auto check by @termux-pacman-bot"
git push origin master
done
fi
fi