geoffrey/eulaurarien
1
0
Fork 0
Code Issues 6 Pull requests Releases 1 Wiki Activity

Use threads not subprocesses

Browse source 
You dumbo
This commit is contained in:
Geoffrey Frogeye 2019-11-14 12:57:06 +01:00
parent 32377229db
commit 08a8eaaada
2 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
fetch_resources.sh
View file

@ -5,7 +5,7 @@
rm -f nameservers rm -f nameservers
touch nameservers touch nameservers
[ -f nameservers.head ] && cat nameservers.head >> nameservers [ -f nameservers.head ] && cat nameservers.head >> nameservers
curl https://public-dns.info/nameservers.txt | sort -R | head -64 >> nameservers curl https://public-dns.info/nameservers.txt | sort -R >> nameservers
# Get top 1M subdomains # Get top 1M subdomains

17
filter_subdomains.py
View file

@ -7,7 +7,8 @@ the ones resolving to a first-party tracker.
""" """
import logging import logging
import multiprocessing import threading
import queue
import os import os
import re import re
import sys import sys
@ -22,15 +23,17 @@ import regexes
DNS_TIMEOUT = 5.0 DNS_TIMEOUT = 5.0
class DnsResolver(multiprocessing.Process): # TODO Try again does not work because sentinel get through first :/
class DnsResolver(threading.Thread):
""" """
Worker process for a DNS resolver. Worker process for a DNS resolver.
Will resolve DNS to match first-party subdomains. Will resolve DNS to match first-party subdomains.
""" """
def __init__(self, def __init__(self,
in_queue: multiprocessing.Queue, in_queue: queue.Queue,
out_queue: multiprocessing.Queue, out_queue: queue.Queue,
server: str): server: str):
super(DnsResolver, self).__init__() super(DnsResolver, self).__init__()
self.log = logging.getLogger(server) self.log = logging.getLogger(server)
@ -92,8 +95,8 @@ class DnsResolver(multiprocessing.Process):
def get_matching_subdomains(subdomains: typing.Iterable[str], def get_matching_subdomains(subdomains: typing.Iterable[str],
nameservers: typing.List[str] = None, nameservers: typing.List[str] = None,
) -> typing.Iterable[typing.Tuple[str, bool]]: ) -> typing.Iterable[typing.Tuple[str, bool]]:
subdomains_queue: multiprocessing.Queue = multiprocessing.Queue() subdomains_queue: queue.Queue = queue.Queue()
results_queue: multiprocessing.Queue = multiprocessing.Queue() results_queue: queue.Queue = queue.Queue()
""" """
Orchestrator of the different DnsResolver threads. Orchestrator of the different DnsResolver threads.
""" """
@ -113,14 +116,12 @@ def get_matching_subdomains(subdomains: typing.Iterable[str],
# sentinel = None ~= EOF # sentinel = None ~= EOF
for _ in servers: for _ in servers:
subdomains_queue.put(None) subdomains_queue.put(None)
subdomains_queue.close()
# Wait for one sentinel per worker # Wait for one sentinel per worker
# In the meantime output results # In the meantime output results
for _ in servers: for _ in servers:
for result in iter(results_queue.get, None): for result in iter(results_queue.get, None):
yield result yield result
results_queue.close()
if __name__ == '__main__': if __name__ == '__main__':