eulaurarien/import_rapid7.sh

#!/usr/bin/env bash

source .env.default
source .env

function log() {
    echo -e "\033[33m$@\033[0m"
}

function api_call {
    curl -s -H "X-Api-Key: $RAPID7_API_KEY" "https://us.api.insight.rapid7.com/opendata/studies/$1/"
}

function get_download_url { # study, dataset
    study="$1"
    dataset="$2"
    if [ -z "$RAPID7_API_KEY" ]
    then
        line=$(curl -s "https://opendata.rapid7.com/$study/" | grep "href=\".\+-$dataset.json.gz\"" | head -1)
        echo "https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"
    else
        filename=$(api_call "$study" | jq '.sonarfile_set[]' -r | grep "${dataset}.json.gz" | sort | tail -1)
        api_call "$study/$filename/download" | jq '.url' -r
    fi
}

function feed_rapid7 { # study, dataset
    study="$1"
    dataset="$2"
    shift; shift
    link="$(get_download_url $study $dataset)"
    log "Reading $dataset dataset from $link…"
    curl -L "$link" | gunzip | ./feed_dns.py rapid7 $@
}

feed_rapid7 sonar.rdns_v2 rdns
feed_rapid7 sonar.fdns_v2 fdns_a --ip4-cache "$CACHE_SIZE"
feed_rapid7 sonar.fdns_v2 fdns_aaaa --ip6-cache "$CACHE_SIZE"
feed_rapid7 sonar.fdns_v2 fdns_cname