2019-12-03 08:48:12 +01:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
2019-12-24 15:08:18 +01:00
|
|
|
source .env.default
|
|
|
|
|
source .env
|
|
|
|
|
|
2019-12-07 18:45:48 +01:00
|
|
|
function log() {
|
|
|
|
|
echo -e "\033[33m$@\033[0m"
|
|
|
|
|
}
|
|
|
|
|
|
2019-12-18 01:03:08 +01:00
|
|
|
log "Compiling nameservers…"
|
|
|
|
|
pv nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list
|
|
|
|
|
|
2019-12-21 19:38:20 +01:00
|
|
|
log "Compiling subdomains…"
|
2019-12-03 15:35:21 +01:00
|
|
|
# Sort by last character to utilize the DNS server caching mechanism
|
2019-12-18 01:03:08 +01:00
|
|
|
# (not as efficient with massdns but it's almost free so why not)
|
|
|
|
|
pv subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list
|
|
|
|
|
|
|
|
|
|
log "Resolving subdomain…"
|
2019-12-25 14:54:57 +01:00
|
|
|
date +%s > "last_updates/massdns.txt"
|
2019-12-26 00:33:23 +01:00
|
|
|
"$MASSDNS_BINARY" --output Snrql --retry REFUSED,SERVFAIL --hashmap-size "$MASSDNS_HASHMAP_SIZE" --resolvers temp/all_nameservers_ip4.list --outfile temp/all_resolved.txt temp/all_subdomains.list
|
2019-12-03 08:48:12 +01:00
|
|
|
|
2019-12-18 01:03:08 +01:00
|
|
|
log "Importing into database…"
|
2019-12-25 15:15:49 +01:00
|
|
|
[ $SINGLE_PROCESS -eq 1 ] && EXTRA_ARGS="--single-process"
|
|
|
|
|
pv temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS
|
