geoffrey/eulaurarien
1
0
Fork 0
Code Issues 6 Pull requests Releases 1 Wiki Activity

Retry failed requests

Browse source
This commit is contained in:
Geoffrey Frogeye 2019-11-14 11:35:05 +01:00
parent 04fe454d99
commit 32377229db
1 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
filter_subdomains.py
View file

@ -22,8 +22,6 @@ import regexes
DNS_TIMEOUT = 5.0 DNS_TIMEOUT = 5.0
# TODO Retry failed requests
class DnsResolver(multiprocessing.Process): class DnsResolver(multiprocessing.Process):
""" """
Worker process for a DNS resolver. Worker process for a DNS resolver.
@ -43,7 +41,7 @@ class DnsResolver(multiprocessing.Process):
self.resolver = dns.resolver.Resolver() self.resolver = dns.resolver.Resolver()
self.resolver.nameservers = [server] self.resolver.nameservers = [server]
def is_subdomain_matching(self, subdomain: str) -> bool: def is_subdomain_matching(self, subdomain: str) -> typing.Optional[bool]:
""" """
Indicates if the subdomain redirects to a first-party tracker. Indicates if the subdomain redirects to a first-party tracker.
""" """
@ -60,10 +58,10 @@ class DnsResolver(multiprocessing.Process):
return False return False
except dns.resolver.NoNameservers: except dns.resolver.NoNameservers:
self.log.warning("All nameservers broken for %s", subdomain) self.log.warning("All nameservers broken for %s", subdomain)
return False return None
except dns.exception.Timeout: except dns.exception.Timeout:
self.log.warning("Timeout for %s", subdomain) self.log.warning("Timeout for %s", subdomain)
return False return None
except dns.name.EmptyLabel: except dns.name.EmptyLabel:
self.log.warning("Empty label for %s", subdomain) self.log.warning("Empty label for %s", subdomain)
return False return False
@ -77,6 +75,13 @@ class DnsResolver(multiprocessing.Process):
self.log.info("Started") self.log.info("Started")
for subdomain in iter(self.in_queue.get, None): for subdomain in iter(self.in_queue.get, None):
matching = self.is_subdomain_matching(subdomain) matching = self.is_subdomain_matching(subdomain)
# If issue, retry
if matching is None:
# matching = False
self.in_queue.put(subdomain)
continue
result = (subdomain, matching) result = (subdomain, matching)
# self.log.debug("%s", result) # self.log.debug("%s", result)
self.out_queue.put(result) self.out_queue.put(result)