Browse Source

Fixed feed_dns not saving in single-threaded mode

Would you believe it, seven hours of processing for nothing
tags/v2.2
Geoffrey Frogeye 2 months ago
parent
commit
018f6548ea
Signed by: geoffrey <geoffrey@frogeye.fr> GPG Key ID: D8A7ECA00A8CD3DD
3 changed files with 7 additions and 4 deletions
  1. +0
    -1
      database.py
  2. +1
    -0
      eulaurarien.sh
  3. +6
    -3
      feed_dns.py

+ 0
- 1
database.py View File

@@ -26,7 +26,6 @@ Level = int


class Path():
# FP add boolean here
pass




+ 1
- 0
eulaurarien.sh View File

@@ -10,4 +10,5 @@
./resolve_subdomains.sh
./import_rapid7.sh
./prune.sh
./export_lists.sh


+ 6
- 3
feed_dns.py View File

@@ -58,6 +58,10 @@ class Writer(multiprocessing.Process):
# IndexError: IP too big
self.log.exception("Cannot execute: %s", record)

def end(self) -> None:
self.db.enter_step('end')
self.db.save()

def run(self) -> None:
self.open_db()
if self.autosave_interval > 0:
@@ -80,9 +84,7 @@ class Writer(multiprocessing.Process):
next_save = time.time() + self.autosave_interval

self.db.enter_step('block_wait')

self.db.enter_step('end')
self.db.save()
self.end()


class Parser():
@@ -240,6 +242,7 @@ if __name__ == '__main__':
)
parser = parser_cls(args.input, writer=writer)
parser.run()
writer.end()
else:
recs_queue: multiprocessing.Queue = multiprocessing.Queue(
maxsize=args.queue_size)


Loading…
Cancel
Save