Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

41 lines
1.4KB

  1. #!/usr/bin/env bash
  2. function log() {
  3. echo -e "\033[33m$@\033[0m"
  4. }
  5. function dl() {
  6. echo "Downloading $1 to $2…"
  7. curl --silent "$1" > "$2"
  8. if [ $? -ne 0 ]
  9. then
  10. echo "Failed!"
  11. fi
  12. }
  13. log "Retrieving rules…"
  14. rm -f rules*/*.cache.*
  15. dl https://easylist.to/easylist/easyprivacy.txt rules_adblock/easyprivacy.cache.txt
  16. log "Retrieving TLD list…"
  17. dl http://data.iana.org/TLD/tlds-alpha-by-domain.txt temp/all_tld.temp.list
  18. grep -v '^#' temp/all_tld.temp.list | awk '{print tolower($0)}' > temp/all_tld.list
  19. log "Retrieving nameservers…"
  20. dl https://public-dns.info/nameservers.txt nameservers/public-dns.cache.list
  21. log "Retrieving top subdomains…"
  22. dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip
  23. unzip top-1m.csv.zip
  24. sed 's|^[0-9]\+,||' top-1m.csv > temp/cisco-umbrella_popularity.fresh.list
  25. rm top-1m.csv top-1m.csv.zip
  26. if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
  27. then
  28. cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
  29. pv -f temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
  30. rm temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list
  31. else
  32. mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list
  33. fi