Skip to content

Commit

Permalink
no fast
Browse files Browse the repository at this point in the history
  • Loading branch information
ipitio committed Nov 6, 2024
1 parent 65c3d3f commit 93885d6
Showing 1 changed file with 7 additions and 7 deletions.
14 changes: 7 additions & 7 deletions src/bkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ main() {
echo >>"$BKG_OWNERS"
awk 'NF' "$BKG_OWNERS" >owners.tmp && mv owners.tmp "$BKG_OWNERS"
sed -i 's/^[[:space:]]*//;s/[[:space:]]*$//' "$BKG_OWNERS"
[[ "$(wc -l <"$BKG_OWNERS")" -ge 100 ]] || seq 1 2 | env_parallel --fast --lb --halt soon,fail=1 page_owner
[[ "$(wc -l <"$BKG_OWNERS")" -ge 100 ]] || seq 1 2 | env_parallel --lb --halt soon,fail=1 page_owner
find "$BKG_INDEX_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort -u | awk '{print $1}' >>"$BKG_OWNERS"
sort -u <<<"$(
explore "$GITHUB_OWNER"
Expand All @@ -112,17 +112,17 @@ main() {
echo "$(
awk -F'|' '{print $1"/"$2}' packages_all
awk -F'|' '{print $2}' packages_all
)" | sort -u | parallel --fast "sed -i '\,^{}$,d' $BKG_OWNERS"
)" | sort -u | parallel "sed -i '\,^{}$,d' $BKG_OWNERS"

if [[ "$pkg_left" == "0" || "${db_size_curr::-1}" == "${db_size_prev::-1}" ]]; then
set_BKG BKG_BATCH_FIRST_STARTED "$today"
rm -f packages_to_update
\cp packages_all packages_to_update
fi

head -n100 "$BKG_OWNERS" | env_parallel --fast --lb save_owner
awk -F'|' '{print $1"/"$2}' packages_to_update | sort -uR 2>/dev/null | head -n1000 | env_parallel --fast --lb save_owner
parallel --fast "sed -i '\,^{}$,d' $BKG_OWNERS" <"$connections"
head -n100 "$BKG_OWNERS" | env_parallel --lb save_owner
awk -F'|' '{print $1"/"$2}' packages_to_update | sort -uR 2>/dev/null | head -n1000 | env_parallel --lb save_owner
parallel "sed -i '\,^{}$,d' $BKG_OWNERS" <"$connections"
set_BKG BKG_DIFF "$db_size_curr"
rm -f "$connections"
else
Expand All @@ -131,7 +131,7 @@ main() {

BKG_BATCH_FIRST_STARTED=$(get_BKG BKG_BATCH_FIRST_STARTED)
[ -d "$BKG_INDEX_DIR" ] || mkdir "$BKG_INDEX_DIR"
get_BKG_set BKG_OWNERS_QUEUE | env_parallel --fast --lb update_owner
get_BKG_set BKG_OWNERS_QUEUE | env_parallel --lb update_owner
sqlite3 "$BKG_INDEX_DB" "select owner_id, owner, repo, package from '$BKG_INDEX_TBL_PKG';" | sort -u >packages_all
echo "Compressing the database..."
sqlite3 "$BKG_INDEX_DB" ".dump" | zstd -22 --ultra --long -T0 -o "$BKG_INDEX_SQL".new.zst
Expand All @@ -146,7 +146,7 @@ main() {
[ ! -f "$older_db" ] || rm -f "$older_db"
mv "$BKG_INDEX_SQL".zst "$older_db"
sqlite3 "$BKG_INDEX_DB" "delete from '$BKG_INDEX_TBL_PKG' where date < '$BKG_BATCH_FIRST_STARTED';"
sqlite3 "$BKG_INDEX_DB" "select name from sqlite_master where type='table' and name like '${BKG_INDEX_TBL_VER}_%';" | parallel --fast --lb "sqlite3 '$BKG_INDEX_DB' 'delete from {} where date < \"$BKG_BATCH_FIRST_STARTED\";'"
sqlite3 "$BKG_INDEX_DB" "select name from sqlite_master where type='table' and name like '${BKG_INDEX_TBL_VER}_%';" | parallel --lb "sqlite3 '$BKG_INDEX_DB' 'delete from {} where date < \"$BKG_BATCH_FIRST_STARTED\";'"
sqlite3 "$BKG_INDEX_DB" "vacuum;"
rm -f "$BKG_INDEX_SQL".new.zst
sqlite3 "$BKG_INDEX_DB" ".dump" | zstd -22 --ultra --long -T0 -o "$BKG_INDEX_SQL".new.zst
Expand Down

0 comments on commit 93885d6

Please sign in to comment.