Script to automatically download from Rapid7 datasets

import_rapid7.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+function log() {
+    echo -e "\033[33m$@\033[0m"
+}
+
+function feed_rapid7_fdns { # dataset
+    dataset=$1
+    line=$(curl -s https://opendata.rapid7.com/sonar.fdns_v2/ | grep "href=\".\+-fdns_$dataset.json.gz\"")
+    link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
+    log "Reading $(echo "$dataset" | awk '{print toupper($0)}') records from $link"
+    curl -L "$link" | gunzip | ./feed_dns.py rapid7
+}
+
+function feed_rapid7_rdns { # dataset
+    dataset=$1
+    line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"")
+    link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
+    log "Reading PTR records from $link"
+    curl -L "$link" | gunzip | ./feed_dns.py rapid7
+}
+
+feed_rapid7_rdns
+feed_rapid7_fdns a
+# feed_rapid7_fdns aaaa
+feed_rapid7_fdns cname

new_workflow.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-function log() {
-    echo -e "\033[33m$@\033[0m"
-}
-
-./fetch_resources.sh
-./import_rules.sh
-
-# TODO Fetch 'em
-log "Reading PTR records…"
-pv ptr.json.gz | gunzip | ./feed_dns.py
-log "Reading A records…"
-pv a.json.gz | gunzip | ./feed_dns.py
-log "Reading CNAME records…"
-pv cname.json.gz | gunzip | ./feed_dns.py
-
-log "Pruning old data…"
-./database.py --prune
-
-./filter_subdomains.sh
-