geoffrey/eulaurarien
1
0
Fork 0
Code Issues 6 Pull requests Releases 1 Wiki Activity

Added more rule sources

Browse source
This commit is contained in:
Geoffrey Frogeye 2019-12-03 17:33:46 +01:00
parent 0159c6037c
commit fe5f0c6c05
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
2 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
fetch_resources.sh
View file

@ -9,12 +9,19 @@ function dl() {
fi fi
} }
# Get rules
echo "Retrieving rules..." > /dev/stderr echo "Retrieving rules..." > /dev/stderr
dl https://easylist.to/easylist/easyprivacy.txt rules_adblock/easyprivacy.cache.txt dl https://easylist.to/easylist/easyprivacy.txt rules_adblock/easyprivacy.cache.txt
# From firebog.net Tracking & Telemetry Lists
dl https://v.firebog.net/hosts/Prigent-Ads.txt rules/prigent-ads.cache.list
dl https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt rules/notrack-blocklist.cache.list
dl https://raw.githubusercontent.com/StevenBlack/hosts/master/data/add.2o7Net/hosts rules_hosts/add2o7.cache.txt dl https://raw.githubusercontent.com/StevenBlack/hosts/master/data/add.2o7Net/hosts rules_hosts/add2o7.cache.txt
dl https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt rules_hosts/spy.cache.txt
dl https://raw.githubusercontent.com/Kees1958/WS3_annual_most_used_survey_blocklist/master/w3tech_hostfile.txt rules/w3tech.cache.list
dl https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt rules_hosts/ads-and-tracking-extended.cache.txt
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt rules_hosts/android-tracking.cache.txt
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt rules_hosts/smart-tv.cache.txt
# dl https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt rules_hosts/amazon-fire-tv.cache.txt
# Get a list of nameservers
echo "Retrieving nameservers..." > /dev/stderr echo "Retrieving nameservers..." > /dev/stderr
rm -f nameservers rm -f nameservers
touch nameservers touch nameservers
@ -23,12 +30,15 @@ dl https://public-dns.info/nameservers.txt nameservers.temp
sort -R nameservers.temp >> nameservers sort -R nameservers.temp >> nameservers
rm nameservers.temp rm nameservers.temp
# Get top 1M subdomains echo "Retrieving top subdomains..." > /dev/stderr
echo "Retrieving subdomains..." > /dev/stderr
dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip
unzip top-1m.csv.zip unzip top-1m.csv.zip
sed 's|^[0-9]\+,||' top-1m.csv > temp/cisco-umbrella_popularity.fresh.list sed 's|^[0-9]\+,||' top-1m.csv > temp/cisco-umbrella_popularity.fresh.list
rm top-1m.csv top-1m.csv.zip rm top-1m.csv top-1m.csv.zip
if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
then
cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
cat temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list pv temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
else
mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list
fi

4
resolve_subdomains.sh
View file

@ -2,9 +2,9 @@
# Resolve the CNAME chain of all the known subdomains for later analysis # Resolve the CNAME chain of all the known subdomains for later analysis
echo "Compiling subdomain lists..." > /dev/stderr echo "Compiling subdomain lists..." > /dev/stderr
cat subdomains/*.list | sort -u > temp/all_subdomains.list pv subdomains/*.list | sort -u > temp/all_subdomains.list
# Sort by last character to utilize the DNS server caching mechanism # Sort by last character to utilize the DNS server caching mechanism
rev temp/all_subdomains.list | sort | rev > temp/all_subdomains_reversort.list pv temp/all_subdomains.list | rev | sort | rev > temp/all_subdomains_reversort.list
./resolve_subdomains.py --input temp/all_subdomains_reversort.list --output temp/all_resolved.csv ./resolve_subdomains.py --input temp/all_subdomains_reversort.list --output temp/all_resolved.csv
sort -u temp/all_resolved.csv > temp/all_resolved_sorted.csv sort -u temp/all_resolved.csv > temp/all_resolved_sorted.csv