From 32377229db3fda8ae55a1aab3967ded928cd38ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 14 Nov 2019 11:35:05 +0100 Subject: [PATCH] Retry failed requests --- filter_subdomains.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/filter_subdomains.py b/filter_subdomains.py index 788b2fc..34682ec 100755 --- a/filter_subdomains.py +++ b/filter_subdomains.py @@ -22,8 +22,6 @@ import regexes DNS_TIMEOUT = 5.0 -# TODO Retry failed requests - class DnsResolver(multiprocessing.Process): """ Worker process for a DNS resolver. @@ -43,7 +41,7 @@ class DnsResolver(multiprocessing.Process): self.resolver = dns.resolver.Resolver() self.resolver.nameservers = [server] - def is_subdomain_matching(self, subdomain: str) -> bool: + def is_subdomain_matching(self, subdomain: str) -> typing.Optional[bool]: """ Indicates if the subdomain redirects to a first-party tracker. """ @@ -60,10 +58,10 @@ class DnsResolver(multiprocessing.Process): return False except dns.resolver.NoNameservers: self.log.warning("All nameservers broken for %s", subdomain) - return False + return None except dns.exception.Timeout: self.log.warning("Timeout for %s", subdomain) - return False + return None except dns.name.EmptyLabel: self.log.warning("Empty label for %s", subdomain) return False @@ -77,6 +75,13 @@ class DnsResolver(multiprocessing.Process): self.log.info("Started") for subdomain in iter(self.in_queue.get, None): matching = self.is_subdomain_matching(subdomain) + + # If issue, retry + if matching is None: + # matching = False + self.in_queue.put(subdomain) + continue + result = (subdomain, matching) # self.log.debug("%s", result) self.out_queue.put(result)