Browse Source

Force pv output

Even if redirected to a file
Allow to see progress when ran in a cron or something
master
Geoffrey Frogeye 2 years ago
parent
commit
171fa93873
Signed by: geoffrey GPG Key ID: D8A7ECA00A8CD3DD
  1. 2
      fetch_resources.sh
  2. 6
      resolve_subdomains.sh

2
fetch_resources.sh

@ -33,7 +33,7 @@ rm top-1m.csv top-1m.csv.zip
if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
then
cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
pv temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
pv -f temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
rm temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list
else
mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list

6
resolve_subdomains.sh

@ -8,12 +8,12 @@ function log() {
}
log "Compiling nameservers…"
pv nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list
pv -f nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list
log "Compiling subdomains…"
# Sort by last character to utilize the DNS server caching mechanism
# (not as efficient with massdns but it's almost free so why not)
pv subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list
pv -f subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list
log "Resolving subdomain…"
date +%s > "last_updates/massdns.txt"
@ -21,4 +21,4 @@ date +%s > "last_updates/massdns.txt"
log "Importing into database…"
[ $SINGLE_PROCESS -eq 1 ] && EXTRA_ARGS="--single-process"
pv temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS
pv -f temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS
Loading…
Cancel
Save