eulaurarien/fetch_resources.sh

41 lines
1.4 KiB
Bash
Raw Normal View History

2019-11-14 11:23:59 +01:00
#!/usr/bin/env bash
2019-12-07 18:45:48 +01:00
function log() {
echo -e "\033[33m$@\033[0m"
}
function dl() {
2019-12-07 18:45:48 +01:00
echo "Downloading $1 to $2"
curl --silent "$1" > "$2"
if [ $? -ne 0 ]
then
echo "Failed!"
fi
}
2019-12-07 18:45:48 +01:00
log "Retrieving rules…"
rm -f rules*/*.cache.*
dl https://easylist.to/easylist/easyprivacy.txt rules_adblock/easyprivacy.cache.txt
log "Retrieving TLD list…"
dl http://data.iana.org/TLD/tlds-alpha-by-domain.txt temp/all_tld.temp.list
grep -v '^#' temp/all_tld.temp.list | awk '{print tolower($0)}' > temp/all_tld.list
2019-12-07 18:45:48 +01:00
log "Retrieving nameservers…"
dl https://public-dns.info/nameservers.txt nameservers/public-dns.cache.list
2019-11-14 11:23:59 +01:00
2019-12-07 18:45:48 +01:00
log "Retrieving top subdomains…"
dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip
2019-11-14 11:23:59 +01:00
unzip top-1m.csv.zip
sed 's|^[0-9]\+,||' top-1m.csv > temp/cisco-umbrella_popularity.fresh.list
2019-11-14 11:23:59 +01:00
rm top-1m.csv top-1m.csv.zip
2019-12-03 17:33:46 +01:00
if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
then
cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
pv -f temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
2019-12-05 19:38:26 +01:00
rm temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list
2019-12-03 17:33:46 +01:00
else
mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list
fi