filter_subdomains.py

@@ -11,18 +11,13 @@ import sys
 import progressbar
 import csv
 import typing
-import ipaddress
 
 # DomainRule = typing.Union[bool, typing.Dict[str, 'DomainRule']]
 DomainRule = typing.Union[bool, typing.Dict]
 
 RULES_DICT: DomainRule = dict()
-RULES_IP: typing.Set[ipaddress.IPv4Network] = set()
-
 
 def subdomain_matching(subdomain: str) -> bool:
-    if not RULES_DICT:
-        return False
     parts = subdomain.split('.')
     parts.reverse()
     dic = RULES_DICT
@@ -35,28 +30,16 @@ def subdomain_matching(subdomain: str) -> bool:
     return False
 
 
-def ip_matching(ip_str: str) -> bool:
-    if not RULES_IP:
-        return False
-    ip = ipaddress.ip_address(ip_str)
-    for net in RULES_IP:
-        if ip in net:
-            return True
-    return False
-
-
 def get_matching(chain: typing.List[str], no_explicit: bool = False
                  ) -> typing.Iterable[str]:
-    if len(chain) <= 1:
-        return
     initial = chain[0]
     cname_destinations = chain[1:-1]
-    a_destination = chain[-1]
+    # a_destination = chain[-1]
     initial_matching = subdomain_matching(initial)
     if no_explicit and initial_matching:
         return
     cname_matching = any(map(subdomain_matching, cname_destinations))
-    if cname_matching or initial_matching or ip_matching(a_destination):
+    if cname_matching or initial_matching:
         yield initial
 
 
@@ -75,14 +58,6 @@ def register_rule(subdomain: str) -> None:
             dic.setdefault(part, dict())
             dic = dic[part]
 
-
-def register_rule_ip(network: str) -> None:
-    net = ipaddress.ip_network(network)
-    RULES_IP.add(net)
-    # If RULES_IP start becoming bigger,
-    # we might implement a binary tree for performance
-
-
 if __name__ == '__main__':
 
     # Parsing arguments
@@ -98,11 +73,8 @@ if __name__ == '__main__':
         '-n', '--no-explicit', action='store_true',
         help="Don't output domains already blocked with rules without CNAME")
     parser.add_argument(
-        '-r', '--rules', type=argparse.FileType('r'),
-        help="List of domains domains to block (with their subdomains)")
-    parser.add_argument(
-        '-p', '--rules-ip', type=argparse.FileType('r'),
-        help="List of IPs ranges to block")
+        '-r', '--rules', type=argparse.FileType('r'), default='rules',
+        help="Rules file")
     args = parser.parse_args()
 
     # Progress bar
@@ -117,12 +89,8 @@ if __name__ == '__main__':
     progress = progressbar.ProgressBar(widgets=widgets)
 
     # Reading rules
-    if args.rules:
-        for rule in args.rules:
-            register_rule(rule.strip())
-    if args.rules_ip:
-        for rule in args.rules_ip:
-            register_rule_ip(rule.strip())
+    for rule in args.rules:
+        register_rule(rule.strip())
 
     # Approximating line count
     if args.input.seekable():

filter_subdomains.sh

@@ -13,23 +13,21 @@ cat rules_adblock/*.txt | grep -v '^!' | grep -v '^\[Adblock' | sort -u > temp/a
 cat rules_hosts/*.txt | grep -v '^#' | grep -v '^$' | cut -d ' ' -f2 > rules/from_hosts.cache.list
 cat rules/*.list | grep -v '^#' | grep -v '^$' | sort -u > temp/all_rules_multi.list
 cat rules/first-party.list | grep -v '^#' | grep -v '^$' | sort -u > temp/all_rules_first.list
-cat rules_ip/*.txt | grep -v '^#' | grep -v '^$' | sort -u > temp/all_ip_rules_multi.txt
-cat rules_ip/first-party.txt | grep -v '^#' | grep -v '^$' | sort -u > temp/all_ip_rules_first.txt
 
 echo "Filtering first-party tracking domains..." > /dev/stderr
-./filter_subdomains.py --rules temp/all_rules_first.list --rules-ip temp/all_ip_rules_first.txt --input temp/all_resolved_sorted.csv --output temp/firstparty-trackers.list
+./filter_subdomains.py --rules temp/all_rules_first.list --input temp/all_resolved_sorted.csv --output temp/firstparty-trackers.list
 sort -u temp/firstparty-trackers.list > dist/firstparty-trackers.txt
 
 echo "Filtering first-party curated tracking domains..." > /dev/stderr
-./filter_subdomains.py --rules temp/all_rules_first.list --rules-ip temp/all_ip_rules_first.txt --input temp/all_resolved_sorted.csv --no-explicit --output temp/firstparty-only-trackers.list
+./filter_subdomains.py --rules temp/all_rules_first.list --input temp/all_resolved_sorted.csv --no-explicit --output temp/firstparty-only-trackers.list
 sort -u temp/firstparty-only-trackers.list > dist/firstparty-only-trackers.txt
 
 echo "Filtering multi-party tracking domains..." > /dev/stderr
-./filter_subdomains.py --rules temp/all_rules_multi.list --rules-ip temp/all_ip_rules_multi.txt --input temp/all_resolved_sorted.csv --output temp/multiparty-trackers.list
+./filter_subdomains.py --rules temp/all_rules_multi.list --input temp/all_resolved_sorted.csv --output temp/multiparty-trackers.list
 sort -u temp/multiparty-trackers.list > dist/multiparty-trackers.txt
 
 echo "Filtering multi-party curated tracking domains..." > /dev/stderr
-./filter_subdomains.py --rules temp/all_rules_multi.list --rules-ip temp/all_ip_rules_multi.txt --input temp/all_resolved_sorted.csv --no-explicit --output temp/multiparty-only-trackers.list
+./filter_subdomains.py --rules temp/all_rules_multi.list --input temp/all_resolved_sorted.csv --no-explicit --output temp/multiparty-only-trackers.list
 sort -u temp/multiparty-only-trackers.list > dist/multiparty-only-trackers.txt
 
 # Format the blocklist so it can be used as a hostlist

rules/first-party.list

@@ -13,5 +13,3 @@ storetail.io
 keyade.com
 # Adobe Experience Cloud
 omtrdc.net
-# ThreatMetrix
-online-metrix.net

rules_ip/.gitignore

@@ -1,2 +0,0 @@
-*.custom.txt
-*.cache.txt

rules_ip/first-party.txt

@@ -1,15 +0,0 @@
-# Eulerian (AS50234 EULERIAN TECHNOLOGIES S.A.S.)
-109.232.192.0/21
-# Criteo (TODO More AS)
-178.250.0.0/21
-91.212.98.0/24
-91.199.242.0/24
-185.235.84.0/24
-# ThreatMetrix (AS30286 ThreatMetrix Inc.)
-69.84.176.0/24
-173.254.179.0/24
-185.32.240.0/23
-185.32.242.0/23
-192.225.156.0/22
-199.101.156.0/23
-199.101.158.0/23