Browse Source

Script to automatically download from Rapid7 datasets

newworkflow
Geoffrey Frogeye 2 years ago
parent
commit
747fe46ad0
Signed by: geoffrey GPG Key ID: D8A7ECA00A8CD3DD
  1. 26
      import_rapid7.sh
  2. 22
      new_workflow.sh

26
import_rapid7.sh

@ -0,0 +1,26 @@
#!/usr/bin/env bash
function log() {
echo -e "\033[33m$@\033[0m"
}
function feed_rapid7_fdns { # dataset
dataset=$1
line=$(curl -s https://opendata.rapid7.com/sonar.fdns_v2/ | grep "href=\".\+-fdns_$dataset.json.gz\"")
link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
log "Reading $(echo "$dataset" | awk '{print toupper($0)}') records from $link"
curl -L "$link" | gunzip | ./feed_dns.py rapid7
}
function feed_rapid7_rdns { # dataset
dataset=$1
line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"")
link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
log "Reading PTR records from $link"
curl -L "$link" | gunzip | ./feed_dns.py rapid7
}
feed_rapid7_rdns
feed_rapid7_fdns a
# feed_rapid7_fdns aaaa
feed_rapid7_fdns cname

22
new_workflow.sh

@ -1,22 +0,0 @@
#!/usr/bin/env bash
function log() {
echo -e "\033[33m$@\033[0m"
}
./fetch_resources.sh
./import_rules.sh
# TODO Fetch 'em
log "Reading PTR records…"
pv ptr.json.gz | gunzip | ./feed_dns.py
log "Reading A records…"
pv a.json.gz | gunzip | ./feed_dns.py
log "Reading CNAME records…"
pv cname.json.gz | gunzip | ./feed_dns.py
log "Pruning old data…"
./database.py --prune
./filter_subdomains.sh
Loading…
Cancel
Save