Browse Source

Fixed feed_dns not saving in single-threaded mode

Would you believe it, seven hours of processing for nothing
master
Geoffrey Frogeye 1 year ago
parent
commit
018f6548ea
Signed by: geoffrey GPG Key ID: D8A7ECA00A8CD3DD
3 changed files with 7 additions and 4 deletions
  1. +0
    -1
      database.py
  2. +1
    -0
      eulaurarien.sh
  3. +6
    -3
      feed_dns.py

+ 0
- 1
database.py View File

@ -26,7 +26,6 @@ Level = int
class Path():
# FP add boolean here
pass


+ 1
- 0
eulaurarien.sh View File

@ -10,4 +10,5 @@
./resolve_subdomains.sh
./import_rapid7.sh
./prune.sh
./export_lists.sh

+ 6
- 3
feed_dns.py View File

@ -58,6 +58,10 @@ class Writer(multiprocessing.Process):
# IndexError: IP too big
self.log.exception("Cannot execute: %s", record)
def end(self) -> None:
self.db.enter_step('end')
self.db.save()
def run(self) -> None:
self.open_db()
if self.autosave_interval > 0:
@ -80,9 +84,7 @@ class Writer(multiprocessing.Process):
next_save = time.time() + self.autosave_interval
self.db.enter_step('block_wait')
self.db.enter_step('end')
self.db.save()
self.end()
class Parser():
@ -240,6 +242,7 @@ if __name__ == '__main__':
)
parser = parser_cls(args.input, writer=writer)
parser.run()
writer.end()
else:
recs_queue: multiprocessing.Queue = multiprocessing.Queue(
maxsize=args.queue_size)


Loading…
Cancel
Save