Skip to content

Update

Update #510

Workflow file for this run

name: "Update"
on:
workflow_dispatch:
schedule:
- cron: '8 */8 * * *' # At minute 8 past every 8th hour
jobs:
updater:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install WHOIS client
run: sudo apt install -y whois
- name: Install JQ
run: sudo apt install -y jq
- name: Install ipcalc
run: sudo apt install -y ipcalc
- name: Set up Python 3.7
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('utils/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Install dependencies
run: |
pip install -r utils/requirements.txt
- name: Download IPs
run: |
set -x
bash tiktok/downloader.sh
sleep 5
bash google/downloader.sh
sleep 5
bash googlecloud/downloader.sh
sleep 5
bash amazon/downloader.sh
sleep 5
bash microsoft/downloader.sh
sleep 5
bash azure/downloader.sh
sleep 5
bash oracle/downloader.sh
sleep 5
bash digitalocean/downloader.sh
sleep 5
bash bing/downloader.sh
sleep 5
bash github/downloader.sh
sleep 5
bash facebook/downloader.sh
sleep 5
bash twitter/downloader.sh
sleep 5
bash linode/downloader.sh
sleep 5
bash vkontakte/downloader.sh
sleep 5
bash yandex/downloader.sh
sleep 5
bash rambler/downloader.sh
sleep 5
bash cloudflare/downloader.sh
sleep 5
bash rostelecom/downloader.sh
sleep 5
bash rugov/downloader.sh
sleep 5
bash qrator/downloader.sh
sleep 5
bash ozonru/downloader.sh
sleep 5
bash alibaba/downloader.sh
sleep 5
bash apple/downloader.sh
sleep 5
bash apple-proxy/downloader.sh
sleep 5
bash akamai/downloader.sh
sleep 5
bash fastly/downloader.sh
sleep 5
bash sber/downloader.sh
sleep 5
bash avito/downloader.sh
sleep 5
bash yandexcloud/downloader.sh
sleep 5
bash mtscloud/downloader.sh
sleep 5
bash mts/downloader.sh
sleep 5
bash corbina/downloader.sh
sleep 5
bash imperva/downloader.sh
sleep 5
bash edgecast/downloader.sh
- name: Create All-In-One ranges
run: |
cat google/ipv4.txt amazon/ipv4.txt microsoft/ipv4.txt oracle/ipv4.txt digitalocean/ipv4.txt bing/ipv4.txt github/ipv4.txt facebook/ipv4.txt twitter/ipv4.txt linode/ipv4.txt cloudflare/ipv4.txt yandex/ipv4.txt vkontakte/ipv4.txt| sort -h | uniq > all/ipv4.txt
cat google/ipv6.txt amazon/ipv6.txt microsoft/ipv6.txt digitalocean/ipv6.txt github/ipv6.txt facebook/ipv6.txt twitter/ipv6.txt linode/ipv6.txt | sort -h | uniq > all/ipv6.txt
- name: Merge Ranges
run: |
set -euo pipefail
set -x
# ipv4
python utils/merge.py --source=google/ipv4.txt | sort -h > google/ipv4_merged.txt
python utils/merge.py --source=tiktok/ipv4.txt | sort -h > tiktok/ipv4_merged.txt
python utils/merge.py --source=googlecloud/ipv4.txt | sort -h > googlecloud/ipv4_merged.txt
python utils/merge.py --source=amazon/ipv4.txt | sort -h > amazon/ipv4_merged.txt
python utils/merge.py --source=microsoft/ipv4.txt | sort -h > microsoft/ipv4_merged.txt
python utils/merge.py --source=oracle/ipv4.txt | sort -h > oracle/ipv4_merged.txt
python utils/merge.py --source=digitalocean/ipv4.txt | sort -h > digitalocean/ipv4_merged.txt
python utils/merge.py --source=bing/ipv4.txt | sort -h > bing/ipv4_merged.txt
python utils/merge.py --source=github/ipv4.txt | sort -h > github/ipv4_merged.txt
python utils/merge.py --source=facebook/ipv4.txt | sort -h > facebook/ipv4_merged.txt
python utils/merge.py --source=twitter/ipv4.txt | sort -h > twitter/ipv4_merged.txt
python utils/merge.py --source=linode/ipv4.txt | sort -h > linode/ipv4_merged.txt
python utils/merge.py --source=vkontakte/ipv4.txt | sort -h >vkontakte/ipv4_merged.txt
python utils/merge.py --source=yandex/ipv4.txt | sort -h >yandex/ipv4_merged.txt
python utils/merge.py --source=rambler/ipv4.txt | sort -h >rambler/ipv4_merged.txt
python utils/merge.py --source=rostelecom/ipv4.txt | sort -h > rostelecom/ipv4_merged.txt
python utils/merge.py --source=rugov/ipv4.txt | sort -h > rugov/ipv4_merged.txt
python utils/merge.py --source=ozonru/ipv4.txt | sort -h > ozonru/ipv4_merged.txt
python utils/merge.py --source=fastly/ipv4.txt | sort -h > fastly/ipv4_merged.txt
python utils/merge.py --source=cloudflare/ipv4.txt | sort -h > cloudflare/ipv4_merged.txt
python utils/merge.py --source=sber/ipv4.txt | sort -h > sber/ipv4_merged.txt
python utils/merge.py --source=azure/ipv4.txt | sort -h > azure/ipv4_merged.txt
python utils/merge.py --source=alibaba/ipv4.txt | sort -h > alibaba/ipv4_merged.txt
python utils/merge.py --source=akamai/ipv4.txt | sort -h > akamai/ipv4_merged.txt
python utils/merge.py --source=avito/ipv4.txt | sort -h > avito/ipv4_merged.txt
python utils/merge.py --source=yandexcloud/ipv4.txt | sort -h > yandexcloud/ipv4_merged.txt
python utils/merge.py --source=mtscloud/ipv4.txt | sort -h > mtscloud/ipv4_merged.txt
python utils/merge.py --source=mts/ipv4.txt | sort -h > mts/ipv4_merged.txt
python utils/merge.py --source=corbina/ipv4.txt | sort -h > corbina/ipv4_merged.txt
python utils/merge.py --source=imperva/ipv4.txt | sort -h > imperva/ipv4_merged.txt
python utils/merge.py --source=edgecast/ipv4.txt | sort -h > edgecast/ipv4_merged.txt
python utils/merge.py --source=apple/ipv4.txt | sort -h > apple/ipv4_merged.txt
python utils/merge.py --source=apple-proxy/ipv4.txt | sort -h > apple-proxy/ipv4_merged.txt
python utils/merge.py --source=all/ipv4.txt | sort -h > all/ipv4_merged.txt
# ipv6
python utils/merge.py --source=google/ipv6.txt | sort -h > google/ipv6_merged.txt
python utils/merge.py --source=googlecloud/ipv6.txt | sort -h > googlecloud/ipv6_merged.txt
python utils/merge.py --source=amazon/ipv6.txt | sort -h > amazon/ipv6_merged.txt
python utils/merge.py --source=microsoft/ipv6.txt | sort -h > microsoft/ipv6_merged.txt
# oracle not provide ipv6
python utils/merge.py --source=digitalocean/ipv6.txt | sort -h > digitalocean/ipv6_merged.txt
# bing not provide ipv6
python utils/merge.py --source=github/ipv6.txt | sort -h > github/ipv6_merged.txt
python utils/merge.py --source=facebook/ipv6.txt | sort -h > facebook/ipv6_merged.txt
python utils/merge.py --source=twitter/ipv6.txt | sort -h > twitter/ipv6_merged.txt
python utils/merge.py --source=linode/ipv6.txt | sort -h > linode/ipv6_merged.txt
python utils/merge.py --source=vkontakte/ipv6.txt | sort -h > vkontakte/ipv6_merged.txt
python utils/merge.py --source=yandex/ipv6.txt | sort -h > yandex/ipv6_merged.txt
python utils/merge.py --source=rambler/ipv6.txt | sort -h > rambler/ipv6_merged.txt
python utils/merge.py --source=apple-proxy/ipv6.txt | sort -h > apple-proxy/ipv6_merged.txt
python utils/merge.py --source=all/ipv6.txt | sort -h > all/ipv6_merged.txt
- name: Commit files
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
git remote add github "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
git pull github ${GITHUB_REF} --ff-only
# Get name & email from 1st commit
git config --local user.email "$(git log --format='%ae' --reverse | head -1)"
git config --local user.name "$(git log --format='%an' --reverse | head -1)"
# try commit
git add .
if [ -z "$(git status --porcelain)" ]; then
echo 'No changes'
exit 0
fi
git commit -m "Auto-update ip ranges"
# push changes
git push github HEAD:${GITHUB_REF}