Added real argument parser

Just so we can have color output when running the script :)
newworkflow_parseropti v1.3.1
Geoffrey Frogeye 2019-11-14 15:37:32 +01:00
parent 88f0bcc648
commit 300fe8e15e
2 changed files with 22 additions and 14 deletions

View File

@ -5,6 +5,7 @@ From a list of subdomains, output only
the ones resolving to a first-party tracker.
"""
import argparse
import logging
import os
import queue
@ -194,11 +195,29 @@ def main() -> None:
Also shows a nice progressbar.
"""
# Initialization
coloredlogs.install(
level='DEBUG',
fmt='%(asctime)s %(name)s %(levelname)s %(message)s'
)
# Parsing arguments
parser = argparse.ArgumentParser(
description="Filter first-party trackers from a list of subdomains")
parser.add_argument(
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
help="Input file with one subdomain per line")
parser.add_argument(
'-o', '--output', type=argparse.FileType('w'), default=sys.stdout,
help="Outptut file with one tracking subdomain per line")
# parser.add_argument(
# '-n', '--nameserver', type=argparse.FileType('r'),
# default='nameservers', help="File with one nameserver per line")
# parser.add_argument(
# '-j', '--workers', type=int, default=512,
# help="Number of threads to use")
args = parser.parse_args()
# Progress bar
widgets = [
progressbar.Percentage(),
@ -210,19 +229,8 @@ def main() -> None:
]
progress = progressbar.ProgressBar(widgets=widgets)
# Parsing arguments
assert len(sys.argv) <= 2
filename = None
if len(sys.argv) == 2 and sys.argv[1] != '-':
filename = sys.argv[1]
progress.max_value = sum(1 for line in open(filename))
textio = open(filename)
else:
textio = sys.stdin
# Cleaning input
iterator = iter(textio)
iterator = iter(args.input)
iterator = map(str.strip, iterator)
iterator = filter(None, iterator)
@ -236,7 +244,7 @@ def main() -> None:
for subdomain, matching in Orchestrator(iterator, servers).run():
progress.update(progress.value + 1)
if matching:
print(subdomain)
print(subdomain, file=args.output)
progress.finish()

View File

@ -3,7 +3,7 @@
# Filter out the subdomains not pointing to a first-party tracker
cat subdomains/*.list | sort -u > temp/all_subdomains.list
./filter_subdomains.py temp/all_subdomains.list > temp/all_toblock.list
./filter_subdomains.py --input temp/all_subdomains.list --output temp/all_toblock.list
sort -u temp/all_toblock.list > dist/firstparty-trackers.txt
# Format the blocklist so it can be used as a hostlist