Compare commits
2 commits
88f0bcc648
...
87bb24c511
Author | SHA1 | Date | |
---|---|---|---|
Geoffrey Frogeye | 87bb24c511 | ||
Geoffrey Frogeye | 300fe8e15e |
|
@ -5,6 +5,7 @@ From a list of subdomains, output only
|
||||||
the ones resolving to a first-party tracker.
|
the ones resolving to a first-party tracker.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import queue
|
import queue
|
||||||
|
@ -194,11 +195,29 @@ def main() -> None:
|
||||||
Also shows a nice progressbar.
|
Also shows a nice progressbar.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Initialization
|
||||||
coloredlogs.install(
|
coloredlogs.install(
|
||||||
level='DEBUG',
|
level='DEBUG',
|
||||||
fmt='%(asctime)s %(name)s %(levelname)s %(message)s'
|
fmt='%(asctime)s %(name)s %(levelname)s %(message)s'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Parsing arguments
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Filter first-party trackers from a list of subdomains")
|
||||||
|
parser.add_argument(
|
||||||
|
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
|
||||||
|
help="Input file with one subdomain per line")
|
||||||
|
parser.add_argument(
|
||||||
|
'-o', '--output', type=argparse.FileType('w'), default=sys.stdout,
|
||||||
|
help="Outptut file with one tracking subdomain per line")
|
||||||
|
# parser.add_argument(
|
||||||
|
# '-n', '--nameserver', type=argparse.FileType('r'),
|
||||||
|
# default='nameservers', help="File with one nameserver per line")
|
||||||
|
# parser.add_argument(
|
||||||
|
# '-j', '--workers', type=int, default=512,
|
||||||
|
# help="Number of threads to use")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Progress bar
|
# Progress bar
|
||||||
widgets = [
|
widgets = [
|
||||||
progressbar.Percentage(),
|
progressbar.Percentage(),
|
||||||
|
@ -210,19 +229,8 @@ def main() -> None:
|
||||||
]
|
]
|
||||||
progress = progressbar.ProgressBar(widgets=widgets)
|
progress = progressbar.ProgressBar(widgets=widgets)
|
||||||
|
|
||||||
# Parsing arguments
|
|
||||||
assert len(sys.argv) <= 2
|
|
||||||
filename = None
|
|
||||||
|
|
||||||
if len(sys.argv) == 2 and sys.argv[1] != '-':
|
|
||||||
filename = sys.argv[1]
|
|
||||||
progress.max_value = sum(1 for line in open(filename))
|
|
||||||
textio = open(filename)
|
|
||||||
else:
|
|
||||||
textio = sys.stdin
|
|
||||||
|
|
||||||
# Cleaning input
|
# Cleaning input
|
||||||
iterator = iter(textio)
|
iterator = iter(args.input)
|
||||||
iterator = map(str.strip, iterator)
|
iterator = map(str.strip, iterator)
|
||||||
iterator = filter(None, iterator)
|
iterator = filter(None, iterator)
|
||||||
|
|
||||||
|
@ -236,7 +244,7 @@ def main() -> None:
|
||||||
for subdomain, matching in Orchestrator(iterator, servers).run():
|
for subdomain, matching in Orchestrator(iterator, servers).run():
|
||||||
progress.update(progress.value + 1)
|
progress.update(progress.value + 1)
|
||||||
if matching:
|
if matching:
|
||||||
print(subdomain)
|
print(subdomain, file=args.output)
|
||||||
progress.finish()
|
progress.finish()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
# Filter out the subdomains not pointing to a first-party tracker
|
# Filter out the subdomains not pointing to a first-party tracker
|
||||||
|
|
||||||
cat subdomains/*.list | sort -u > temp/all_subdomains.list
|
cat subdomains/*.list | sort -u > temp/all_subdomains.list
|
||||||
./filter_subdomains.py temp/all_subdomains.list > temp/all_toblock.list
|
./filter_subdomains.py --input temp/all_subdomains.list --output temp/all_toblock.list
|
||||||
sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
|
sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
|
||||||
|
|
||||||
# Format the blocklist so it can be used as a hostlist
|
# Format the blocklist so it can be used as a hostlist
|
||||||
|
@ -21,7 +21,7 @@ sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
|
||||||
echo "# Number of source subdomains: $(wc -l temp/all_subdomains.list | cut -d' ' -f1)"
|
echo "# Number of source subdomains: $(wc -l temp/all_subdomains.list | cut -d' ' -f1)"
|
||||||
echo "# Number of known trackers : $(python -c 'import regexes; print(len(regexes.REGEXES))')"
|
echo "# Number of known trackers : $(python -c 'import regexes; print(len(regexes.REGEXES))')"
|
||||||
echo "# Number of blocked subdomains: $(wc -l dist/firstparty-trackers.txt | cut -d' ' -f1)"
|
echo "# Number of blocked subdomains: $(wc -l dist/firstparty-trackers.txt | cut -d' ' -f1)"
|
||||||
echo "# Number of first-party subdomains: $(./filter_out_explicit.py dist/firstparty-trackers.txt | wc)"
|
echo "# Number of first-party subdomains: $(./filter_out_explicit.py dist/firstparty-trackers.txt | wc -l)"
|
||||||
echo
|
echo
|
||||||
cat dist/firstparty-trackers.txt | while read host;
|
cat dist/firstparty-trackers.txt | while read host;
|
||||||
do
|
do
|
||||||
|
|
Loading…
Reference in a new issue