|
|
@ -184,35 +184,40 @@ if __name__ == '__main__': |
|
|
|
# Parsing arguments |
|
|
|
log = logging.getLogger('feed_dns') |
|
|
|
args_parser = argparse.ArgumentParser( |
|
|
|
description="TODO") |
|
|
|
description="Read DNS records and import " |
|
|
|
"tracking-relevant data into the database") |
|
|
|
args_parser.add_argument( |
|
|
|
'parser', |
|
|
|
choices=PARSERS.keys(), |
|
|
|
help="TODO") |
|
|
|
help="Input format") |
|
|
|
args_parser.add_argument( |
|
|
|
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin, |
|
|
|
help="TODO") |
|
|
|
args_parser.add_argument( |
|
|
|
'-j', '--workers', type=int, default=4, |
|
|
|
help="TODO") |
|
|
|
help="Input file") |
|
|
|
args_parser.add_argument( |
|
|
|
'-b', '--block-size', type=int, default=1024, |
|
|
|
help="TODO") |
|
|
|
help="Performance tuning value") |
|
|
|
args_parser.add_argument( |
|
|
|
'-q', '--queue-size', type=int, default=128, |
|
|
|
help="TODO") |
|
|
|
help="Performance tuning value") |
|
|
|
args_parser.add_argument( |
|
|
|
'-a', '--autosave-interval', type=int, default=900, |
|
|
|
help="TODO seconds") |
|
|
|
help="Interval to which the database will save in seconds. " |
|
|
|
"0 to disable.") |
|
|
|
args_parser.add_argument( |
|
|
|
'-4', '--ip4-cache', type=int, default=0, |
|
|
|
help="TODO bytes max 512 MiB") |
|
|
|
help="RAM cache for faster IPv4 lookup. " |
|
|
|
"Maximum useful value: 512 MiB (536870912). " |
|
|
|
"Warning: Depending on the rules, this might already " |
|
|
|
"be a memory-heavy process, even without the cache.") |
|
|
|
args = args_parser.parse_args() |
|
|
|
|
|
|
|
recs_queue: multiprocessing.Queue = multiprocessing.Queue( |
|
|
|
maxsize=args.queue_size) |
|
|
|
maxsize=args.queue_size) |
|
|
|
|
|
|
|
writer = Writer(recs_queue, autosave_interval=args.autosave_interval, ip4_cache=args.ip4_cache) |
|
|
|
writer = Writer(recs_queue, |
|
|
|
autosave_interval=args.autosave_interval, |
|
|
|
ip4_cache=args.ip4_cache |
|
|
|
) |
|
|
|
writer.start() |
|
|
|
|
|
|
|
parser = PARSERS[args.parser](args.input, recs_queue, args.block_size) |
|
|
|