geoffrey/eulaurarien
1
0
Fork 0
Code Issues 6 Pull requests Releases 1 Wiki Activity

Script to automatically download from Rapid7 datasets

Browse source
This commit is contained in:
Geoffrey Frogeye 2019-12-17 15:04:19 +01:00
parent b43cb1725c
commit 747fe46ad0
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
2 changed files with 26 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
import_rapid7.sh Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
function log() {
echo -e "\033[33m$@\033[0m"
}
function feed_rapid7_fdns { # dataset
dataset=$1
line=$(curl -s https://opendata.rapid7.com/sonar.fdns_v2/ | grep "href=\".\+-fdns_$dataset.json.gz\"")
link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
log "Reading $(echo "$dataset" | awk '{print toupper($0)}') records from $link"
curl -L "$link" | gunzip | ./feed_dns.py rapid7
}
function feed_rapid7_rdns { # dataset
dataset=$1
line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"")
link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
log "Reading PTR records from $link"
curl -L "$link" | gunzip | ./feed_dns.py rapid7
}
feed_rapid7_rdns
feed_rapid7_fdns a
# feed_rapid7_fdns aaaa
feed_rapid7_fdns cname

22
new_workflow.sh
View file

@ -1,22 +0,0 @@
#!/usr/bin/env bash
function log() {
echo -e "\033[33m$@\033[0m"
}
./fetch_resources.sh
./import_rules.sh
# TODO Fetch 'em
log "Reading PTR records…"
pv ptr.json.gz | gunzip | ./feed_dns.py
log "Reading A records…"
pv a.json.gz | gunzip | ./feed_dns.py
log "Reading CNAME records…"
pv cname.json.gz | gunzip | ./feed_dns.py
log "Pruning old data…"
./database.py --prune
./filter_subdomains.sh