Browse Source

Retry failed requests

newworkflow_parseropti v1.3.0
Geoffrey Frogeye 3 years ago
parent
commit
32377229db
  1. 15
      filter_subdomains.py

15
filter_subdomains.py

@ -22,8 +22,6 @@ import regexes
DNS_TIMEOUT = 5.0
# TODO Retry failed requests
class DnsResolver(multiprocessing.Process):
"""
Worker process for a DNS resolver.
@ -43,7 +41,7 @@ class DnsResolver(multiprocessing.Process):
self.resolver = dns.resolver.Resolver()
self.resolver.nameservers = [server]
def is_subdomain_matching(self, subdomain: str) -> bool:
def is_subdomain_matching(self, subdomain: str) -> typing.Optional[bool]:
"""
Indicates if the subdomain redirects to a first-party tracker.
"""
@ -60,10 +58,10 @@ class DnsResolver(multiprocessing.Process):
return False
except dns.resolver.NoNameservers:
self.log.warning("All nameservers broken for %s", subdomain)
return False
return None
except dns.exception.Timeout:
self.log.warning("Timeout for %s", subdomain)
return False
return None
except dns.name.EmptyLabel:
self.log.warning("Empty label for %s", subdomain)
return False
@ -77,6 +75,13 @@ class DnsResolver(multiprocessing.Process):
self.log.info("Started")
for subdomain in iter(self.in_queue.get, None):
matching = self.is_subdomain_matching(subdomain)
# If issue, retry
if matching is None:
# matching = False
self.in_queue.put(subdomain)
continue
result = (subdomain, matching)
# self.log.debug("%s", result)
self.out_queue.put(result)

Loading…
Cancel
Save