From 747fe46ad0a13fc255fa3a4e114f1e5c1d8a9e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?=
 <geoffrey@frogeye.fr>
Date: Tue, 17 Dec 2019 15:04:19 +0100
Subject: [PATCH] Script to automatically download from Rapid7 datasets

---
 import_rapid7.sh | 26 ++++++++++++++++++++++++++
 new_workflow.sh  | 22 ----------------------
 2 files changed, 26 insertions(+), 22 deletions(-)
 create mode 100755 import_rapid7.sh
 delete mode 100755 new_workflow.sh

diff --git a/import_rapid7.sh b/import_rapid7.sh
new file mode 100755
index 0000000..c8eacd1
--- /dev/null
+++ b/import_rapid7.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+function log() {
+    echo -e "\033[33m$@\033[0m"
+}
+
+function feed_rapid7_fdns { # dataset
+    dataset=$1
+    line=$(curl -s https://opendata.rapid7.com/sonar.fdns_v2/ | grep "href=\".\+-fdns_$dataset.json.gz\"")
+    link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
+    log "Reading $(echo "$dataset" | awk '{print toupper($0)}') records from $link"
+    curl -L "$link" | gunzip | ./feed_dns.py rapid7
+}
+
+function feed_rapid7_rdns { # dataset
+    dataset=$1
+    line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"")
+    link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
+    log "Reading PTR records from $link"
+    curl -L "$link" | gunzip | ./feed_dns.py rapid7
+}
+
+feed_rapid7_rdns
+feed_rapid7_fdns a
+# feed_rapid7_fdns aaaa
+feed_rapid7_fdns cname
diff --git a/new_workflow.sh b/new_workflow.sh
deleted file mode 100755
index c98cd46..0000000
--- a/new_workflow.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-function log() {
-    echo -e "\033[33m$@\033[0m"
-}
-
-./fetch_resources.sh
-./import_rules.sh
-
-# TODO Fetch 'em
-log "Reading PTR records…"
-pv ptr.json.gz | gunzip | ./feed_dns.py
-log "Reading A records…"
-pv a.json.gz | gunzip | ./feed_dns.py
-log "Reading CNAME records…"
-pv cname.json.gz | gunzip | ./feed_dns.py
-
-log "Pruning old data…"
-./database.py --prune
-
-./filter_subdomains.sh
-