diff --git a/import_rapid7.sh b/import_rapid7.sh new file mode 100755 index 0000000..c8eacd1 --- /dev/null +++ b/import_rapid7.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +function log() { + echo -e "\033[33m$@\033[0m" +} + +function feed_rapid7_fdns { # dataset + dataset=$1 + line=$(curl -s https://opendata.rapid7.com/sonar.fdns_v2/ | grep "href=\".\+-fdns_$dataset.json.gz\"") + link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)" + log "Reading $(echo "$dataset" | awk '{print toupper($0)}') records from $link" + curl -L "$link" | gunzip | ./feed_dns.py rapid7 +} + +function feed_rapid7_rdns { # dataset + dataset=$1 + line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"") + link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)" + log "Reading PTR records from $link" + curl -L "$link" | gunzip | ./feed_dns.py rapid7 +} + +feed_rapid7_rdns +feed_rapid7_fdns a +# feed_rapid7_fdns aaaa +feed_rapid7_fdns cname diff --git a/new_workflow.sh b/new_workflow.sh deleted file mode 100755 index c98cd46..0000000 --- a/new_workflow.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -function log() { - echo -e "\033[33m$@\033[0m" -} - -./fetch_resources.sh -./import_rules.sh - -# TODO Fetch 'em -log "Reading PTR records…" -pv ptr.json.gz | gunzip | ./feed_dns.py -log "Reading A records…" -pv a.json.gz | gunzip | ./feed_dns.py -log "Reading CNAME records…" -pv cname.json.gz | gunzip | ./feed_dns.py - -log "Pruning old data…" -./database.py --prune - -./filter_subdomains.sh -