Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

25 lines
922B

  1. #!/usr/bin/env bash
  2. source .env.default
  3. source .env
  4. function log() {
  5. echo -e "\033[33m$@\033[0m"
  6. }
  7. log "Compiling nameservers…"
  8. pv -f nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list
  9. log "Compiling subdomains…"
  10. # Sort by last character to utilize the DNS server caching mechanism
  11. # (not as efficient with massdns but it's almost free so why not)
  12. pv -f subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list
  13. log "Resolving subdomain…"
  14. date +%s > "last_updates/massdns.txt"
  15. "$MASSDNS_BINARY" --output Snrql --hashmap-size "$MASSDNS_HASHMAP_SIZE" --resolvers temp/all_nameservers_ip4.list --outfile temp/all_resolved.txt temp/all_subdomains.list
  16. log "Importing into database…"
  17. [ $SINGLE_PROCESS -eq 1 ] && EXTRA_ARGS="--single-process"
  18. pv -f temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS