Browse Source

Force pv output

Even if redirected to a file
Allow to see progress when ran in a cron or something
tags/v2.2
Geoffrey Frogeye 3 months ago
parent
commit
171fa93873
Signed by: geoffrey <geoffrey@frogeye.fr> GPG Key ID: D8A7ECA00A8CD3DD
2 changed files with 4 additions and 4 deletions
  1. +1
    -1
      fetch_resources.sh
  2. +3
    -3
      resolve_subdomains.sh

+ 1
- 1
fetch_resources.sh View File

@@ -33,7 +33,7 @@ rm top-1m.csv top-1m.csv.zip
if [ -f subdomains/cisco-umbrella_popularity.cache.list ]
then
cp subdomains/cisco-umbrella_popularity.cache.list temp/cisco-umbrella_popularity.old.list
pv temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
pv -f temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list | sort -u > subdomains/cisco-umbrella_popularity.cache.list
rm temp/cisco-umbrella_popularity.old.list temp/cisco-umbrella_popularity.fresh.list
else
mv temp/cisco-umbrella_popularity.fresh.list subdomains/cisco-umbrella_popularity.cache.list


+ 3
- 3
resolve_subdomains.sh View File

@@ -8,12 +8,12 @@ function log() {
}

log "Compiling nameservers…"
pv nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list
pv -f nameservers/*.list | ./validate_list.py --ip4 | sort -u > temp/all_nameservers_ip4.list

log "Compiling subdomains…"
# Sort by last character to utilize the DNS server caching mechanism
# (not as efficient with massdns but it's almost free so why not)
pv subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list
pv -f subdomains/*.list | ./validate_list.py --domain | rev | sort -u | rev > temp/all_subdomains.list

log "Resolving subdomain…"
date +%s > "last_updates/massdns.txt"
@@ -21,4 +21,4 @@ date +%s > "last_updates/massdns.txt"

log "Importing into database…"
[ $SINGLE_PROCESS -eq 1 ] && EXTRA_ARGS="--single-process"
pv temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS
pv -f temp/all_resolved.txt | ./feed_dns.py massdns --ip4-cache "$CACHE_SIZE" $EXTRA_ARGS

Loading…
Cancel
Save