2019-11-14 11:23:59 +01:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
2019-12-07 18:45:48 +01:00
|
|
|
function log() {
|
|
|
|
|
echo -e "\033[33m$@\033[0m"
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-03 08:48:12 +01:00
|
|
|
function dl() {
|
2019-12-07 18:45:48 +01:00
|
|
|
echo "Downloading $1 to $2…"
|
2019-12-03 08:48:12 +01:00
|
|
|
curl --silent "$1" > "$2"
|
|
|
|
|
if [ $? -ne 0 ]
|
|
|
|
|
then
|
|
|
|
|
echo "Failed!"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-07 18:45:48 +01:00
|
|
|
|
|
|
|
|
log "Retrieving rules…"
|
2019-12-03 21:25:20 +01:00
|
|
|
rm -f rules*/*.cache.*
|
2019-12-03 08:48:12 +01:00
|
|
|
dl https://easylist.to/easylist/easyprivacy.txt rules_adblock/easyprivacy.cache.txt
|
2019-12-03 17:33:46 +01:00
|
|
|
# From firebog.net Tracking & Telemetry Lists
|
2019-12-14 16:04:19 +01:00
|
|
|
# dl https://v.firebog.net/hosts/Prigent-Ads.txt rules/prigent-ads.cache.list
|
2019-12-05 00:50:32 +01:00
|
|
|
# dl https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt rules/notrack-blocklist.cache.list
|
|
|
|
|
# False positives: https://github.com/WaLLy3K/wally3k.github.io/issues/73 -> 69.media.tumblr.com chicdn.net
|
2019-12-03 08:48:12 +01:00
|
|
|
dl https://raw.githubusercontent.com/StevenBlack/hosts/master/data/add.2o7Net/hosts rules_hosts/add2o7.cache.txt
|
2019-12-03 17:33:46 +01:00
|
|
|
dl https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt rules_hosts/spy.cache.txt
|
2019-12-05 00:50:32 +01:00
|
|
|
# dl https://raw.githubusercontent.com/Kees1958/WS3_annual_most_used_survey_blocklist/master/w3tech_hostfile.txt rules/w3tech.cache.list
|
|
|
|
|
# False positives: agreements.apple.com -> edgekey.net
|
2019-12-03 21:25:20 +01:00
|
|
|
# dl https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt rules_hosts/ads-and-tracking-extended.cache.txt # Lots of false-positives
|
2019-12-03 17:33:46 +01:00
|
|
|
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt rules_hosts/android-tracking.cache.txt
|
|
|
|
|
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt rules_hosts/smart-tv.cache.txt
|
|
|
|
|
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt rules_hosts/amazon-fire-tv.cache.txt
|
2019-11-15 08:57:31 +01:00
|
|
|
|
2019-12-17 19:53:05 +01:00
|
|
|
log "Retrieving TLD list…"
|
|
|
|
|
dl http://data.iana.org/TLD/tlds-alpha-by-domain.txt temp/all_tld.temp.list
|
|
|
|
|
grep -v '^#' temp/all_tld.temp.list | awk '{print tolower($0)}' > temp/all_tld.list
|
|
|
|
|
|
2019-12-07 18:45:48 +01:00
|
|
|
log "Retrieving nameservers…"
|
2019-12-18 01:03:08 +01:00
|
|
|
dl https://public-dns.info/nameservers.txt nameservers/public-dns.list
|
2019-11-14 11:23:59 +01:00
|
|
|
|
2019-12-07 18:45:48 +01:00
|
|
|
log "Retrieving top subdomains…"
|
2019-12-03 09:02:59 +01:00
|
|
|
dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip
|
2019-11-14 11:23:59 +01:00
|
|
|
unzip top-1m.csv.zip
|
2019-12-03 09:02:59 +01:00
|
|
|
sed 's|^[0-9]\+,||' top-1m.csv > temp/cisco-umbrella_popularity.fresh.list
|
2019-11-14 11:23:59 +01:00
|
|
|
rm top-1m.csv top-1m.csv.zip
|
2019-12-03 17:33:46 +01:00
|
|
|
if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
|
|
|
|
|
then
|
|
|
|
|
cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
|
|
|
|
|
pv temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
|
2019-12-05 19:38:26 +01:00
|
|
|
rm temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list
|
2019-12-03 17:33:46 +01:00
|
|
|
else
|
|
|
|
|
mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list
|
|
|
|
|
fi
|
