filter_subdomains.py

@@ -5,7 +5,6 @@ From a list of subdomains, output only
 the ones resolving to a first-party tracker.
 """
 
-import argparse
 import logging
 import os
 import queue
@@ -195,29 +194,11 @@ def main() -> None:
     Also shows a nice progressbar.
     """
 
-    # Initialization
     coloredlogs.install(
         level='DEBUG',
         fmt='%(asctime)s %(name)s %(levelname)s %(message)s'
     )
 
-    # Parsing arguments
-    parser = argparse.ArgumentParser(
-        description="Filter first-party trackers from a list of subdomains")
-    parser.add_argument(
-        '-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
-        help="Input file with one subdomain per line")
-    parser.add_argument(
-        '-o', '--output', type=argparse.FileType('w'), default=sys.stdout,
-        help="Outptut file with one tracking subdomain per line")
-    # parser.add_argument(
-    #     '-n', '--nameserver', type=argparse.FileType('r'),
-    #     default='nameservers', help="File with one nameserver per line")
-    # parser.add_argument(
-    #     '-j', '--workers', type=int, default=512,
-    #     help="Number of threads to use")
-    args = parser.parse_args()
-
     # Progress bar
     widgets = [
         progressbar.Percentage(),
@@ -229,8 +210,19 @@ def main() -> None:
     ]
     progress = progressbar.ProgressBar(widgets=widgets)
 
+    # Parsing arguments
+    assert len(sys.argv) <= 2
+    filename = None
+
+    if len(sys.argv) == 2 and sys.argv[1] != '-':
+        filename = sys.argv[1]
+        progress.max_value = sum(1 for line in open(filename))
+        textio = open(filename)
+    else:
+        textio = sys.stdin
+
     # Cleaning input
-    iterator = iter(args.input)
+    iterator = iter(textio)
     iterator = map(str.strip, iterator)
     iterator = filter(None, iterator)
 
@@ -244,7 +236,7 @@ def main() -> None:
     for subdomain, matching in Orchestrator(iterator, servers).run():
         progress.update(progress.value + 1)
         if matching:
-            print(subdomain, file=args.output)
+            print(subdomain)
     progress.finish()
 
 

filter_subdomains.sh

@@ -3,7 +3,7 @@
 # Filter out the subdomains not pointing to a first-party tracker
 
 cat subdomains/*.list | sort -u > temp/all_subdomains.list
-./filter_subdomains.py --input temp/all_subdomains.list --output temp/all_toblock.list
+./filter_subdomains.py temp/all_subdomains.list > temp/all_toblock.list
 sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
 
 # Format the blocklist so it can be used as a hostlist
@@ -21,7 +21,7 @@ sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
     echo "# Number of source subdomains: $(wc -l temp/all_subdomains.list | cut -d' ' -f1)"
     echo "# Number of known trackers : $(python -c 'import regexes; print(len(regexes.REGEXES))')"
     echo "# Number of blocked subdomains: $(wc -l dist/firstparty-trackers.txt | cut -d' ' -f1)"
-    echo "# Number of first-party subdomains: $(./filter_out_explicit.py dist/firstparty-trackers.txt | wc -l)"
+    echo "# Number of first-party subdomains: $(./filter_out_explicit.py dist/firstparty-trackers.txt | wc)"
     echo
     cat dist/firstparty-trackers.txt | while read host;
     do