Allow selecting region for DNS resolvers

massdns data import should work with incomplete files
2024-12-09 19:38:14 +01:00 · 2024-12-09 19:36:52 +01:00
3 changed files with 6 additions and 2 deletions
--- a/.env.default
+++ b/.env.default
@ -3,3 +3,4 @@ MASSDNS_HASHMAP_SIZE=1000
 PROFILE=0
 SINGLE_PROCESS=0
 MASSDNS_BINARY=massdns
+RESOLVERS_REGION=us
--- a/feed_dns.py
+++ b/feed_dns.py
@ -167,7 +167,7 @@ class MassDnsParser(Parser):
                    )
                    self.register(record)
                    self.prof.enter_step("parse_massdns")
-            except KeyError:
+            except IndexError:
                continue


--- a/fetch_resources.sh
+++ b/fetch_resources.sh
@ -1,5 +1,8 @@
 #!/usr/bin/env bash

+source .env.default
+source .env
+
 function log() {
    echo -e "\033[33m$@\033[0m"
 }
@ -28,7 +31,7 @@ dl http://data.iana.org/TLD/tlds-alpha-by-domain.txt temp/all_tld.temp.list
 grep -v '^#' temp/all_tld.temp.list | awk '{print tolower($0)}' > temp/all_tld.list

 log "Retrieving nameservers…"
-dl https://public-dns.info/nameservers.txt nameservers/public-dns.cache.list
+dl https://public-dns.info/nameserver/${RESOLVERS_REGION}.txt nameservers/public-dns.cache.list

 log "Retrieving top subdomains…"
 dl http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip top-1m.csv.zip
Author	SHA1	Message	Date
Geoffrey Frogeye	d8a3e2ac9c	Allow selecting region for DNS resolvers	2024-12-09 19:38:14 +01:00
Geoffrey Frogeye	3acffde75a	massdns data import should work with incomplete files	2024-12-09 19:36:52 +01:00