Fixed feed_dns not saving in single-threaded mode

Would you believe it, seven hours of processing for nothing
master
Geoffrey Frogeye 2019-12-26 00:02:01 +01:00
bovenliggende 0b9e2d0975
commit 018f6548ea
Getekend door: geoffrey
GPG sleutel-ID: D8A7ECA00A8CD3DD
3 gewijzigde bestanden met toevoegingen van 7 en 4 verwijderingen

Bestand weergeven

@ -26,7 +26,6 @@ Level = int
class Path():
# FP add boolean here
pass

Bestand weergeven

@ -10,4 +10,5 @@
./resolve_subdomains.sh
./import_rapid7.sh
./prune.sh
./export_lists.sh

Bestand weergeven

@ -58,6 +58,10 @@ class Writer(multiprocessing.Process):
# IndexError: IP too big
self.log.exception("Cannot execute: %s", record)
def end(self) -> None:
self.db.enter_step('end')
self.db.save()
def run(self) -> None:
self.open_db()
if self.autosave_interval > 0:
@ -80,9 +84,7 @@ class Writer(multiprocessing.Process):
next_save = time.time() + self.autosave_interval
self.db.enter_step('block_wait')
self.db.enter_step('end')
self.db.save()
self.end()
class Parser():
@ -240,6 +242,7 @@ if __name__ == '__main__':
)
parser = parser_cls(args.input, writer=writer)
parser.run()
writer.end()
else:
recs_queue: multiprocessing.Queue = multiprocessing.Queue(
maxsize=args.queue_size)