geoffrey/eulaurarien
1
0
Fork 0
Code Issues 6 Pull requests Releases 1 Wiki Activity

Fixed feed_dns not saving in single-threaded mode

Browse source 
Would you believe it, seven hours of processing for nothing
This commit is contained in:
Geoffrey Frogeye 2019-12-26 00:02:01 +01:00
parent 0b9e2d0975
commit 018f6548ea
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
3 changed files with 7 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
database.py
View file

@ -26,7 +26,6 @@ Level = int
class Path(): class Path():
# FP add boolean here
pass pass

1
eulaurarien.sh
View file

@ -10,4 +10,5 @@
./resolve_subdomains.sh ./resolve_subdomains.sh
./import_rapid7.sh ./import_rapid7.sh
./prune.sh ./prune.sh
./export_lists.sh

9
feed_dns.py
View file

@ -58,6 +58,10 @@ class Writer(multiprocessing.Process):
# IndexError: IP too big # IndexError: IP too big
self.log.exception("Cannot execute: %s", record) self.log.exception("Cannot execute: %s", record)
def end(self) -> None:
self.db.enter_step('end')
self.db.save()
def run(self) -> None: def run(self) -> None:
self.open_db() self.open_db()
if self.autosave_interval > 0: if self.autosave_interval > 0:
@ -80,9 +84,7 @@ class Writer(multiprocessing.Process):
next_save = time.time() + self.autosave_interval next_save = time.time() + self.autosave_interval
self.db.enter_step('block_wait') self.db.enter_step('block_wait')
self.end()
self.db.enter_step('end')
self.db.save()
class Parser(): class Parser():
@ -240,6 +242,7 @@ if __name__ == '__main__':
) )
parser = parser_cls(args.input, writer=writer) parser = parser_cls(args.input, writer=writer)
parser.run() parser.run()
writer.end()
else: else:
recs_queue: multiprocessing.Queue = multiprocessing.Queue( recs_queue: multiprocessing.Queue = multiprocessing.Queue(
maxsize=args.queue_size) maxsize=args.queue_size)