Browse Source

Removed TODO placeholders in commands description

It's better than nothing but not by that much
tags/v2.1
Geoffrey Frogeye 2 months ago
parent
commit
53b14c6ffa
Signed by: geoffrey <geoffrey@frogeye.fr> GPG Key ID: D8A7ECA00A8CD3DD
7 changed files with 47 additions and 33 deletions
  1. +5
    -3
      db.py
  2. +14
    -9
      export.py
  3. +1
    -1
      feed_asn.py
  4. +17
    -12
      feed_dns.py
  5. +5
    -3
      feed_rules.py
  6. +1
    -1
      import_rapid7.sh
  7. +4
    -4
      validate_list.py

+ 5
- 3
db.py View File

@@ -18,14 +18,16 @@ if __name__ == '__main__':
help="Remove old entries from database")
parser.add_argument(
'-b', '--prune-base', action='store_true',
help="TODO")
help="With --prune, only prune base rules "
"(the ones added by ./feed_rules.py)")
parser.add_argument(
'-s', '--prune-before', type=int,
default=(int(time.time()) - 60*60*24*31*6),
help="TODO")
help="With --prune, only rules updated before "
"this UNIX timestamp will be deleted")
parser.add_argument(
'-r', '--references', action='store_true',
help="Update the reference count")
help="DEBUG: Update the reference count")
args = parser.parse_args()

if not args.initialize:


+ 14
- 9
export.py View File

@@ -9,31 +9,36 @@ if __name__ == '__main__':

# Parsing arguments
parser = argparse.ArgumentParser(
description="TODO")
description="Export the hostnames rules stored "
"in the Database as plain text")
parser.add_argument(
'-o', '--output', type=argparse.FileType('w'), default=sys.stdout,
help="TODO")
help="Output file, one rule per line")
parser.add_argument(
'-f', '--first-party', action='store_true',
help="TODO")
help="Only output rules issued from first-party sources")
parser.add_argument(
'-e', '--end-chain', action='store_true',
help="TODO")
help="Only output rules that are not referenced by any other")
parser.add_argument(
'-r', '--rules', action='store_true',
help="TODO")
help="Output all kinds of rules, not just hostnames")
parser.add_argument(
'-b', '--base-rules', action='store_true',
help="TODO implies rules")
help="Output base rules "
"(the ones added by ./feed_rules.py) "
"(implies --rules)")
parser.add_argument(
'-d', '--no-dupplicates', action='store_true',
help="TODO")
help="Do not output rules that already match a zone/network rule "
"(e.g. dummy.example.com when there's a zone example.com rule)")
parser.add_argument(
'-x', '--explain', action='store_true',
help="TODO")
help="Show the chain of rules leading to one "
"(and the number of references they have)")
parser.add_argument(
'-c', '--count', action='store_true',
help="TODO")
help="Show the number of rules per type instead of listing them")
args = parser.parse_args()

DB = database.Database()


+ 1
- 1
feed_asn.py View File

@@ -36,7 +36,7 @@ if __name__ == '__main__':

# Parsing arguments
parser = argparse.ArgumentParser(
description="TODO")
description="Add the IP ranges associated to the AS in the database")
args = parser.parse_args()

DB = database.Database()


+ 17
- 12
feed_dns.py View File

@@ -184,35 +184,40 @@ if __name__ == '__main__':
# Parsing arguments
log = logging.getLogger('feed_dns')
args_parser = argparse.ArgumentParser(
description="TODO")
description="Read DNS records and import "
"tracking-relevant data into the database")
args_parser.add_argument(
'parser',
choices=PARSERS.keys(),
help="TODO")
help="Input format")
args_parser.add_argument(
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
help="TODO")
args_parser.add_argument(
'-j', '--workers', type=int, default=4,
help="TODO")
help="Input file")
args_parser.add_argument(
'-b', '--block-size', type=int, default=1024,
help="TODO")
help="Performance tuning value")
args_parser.add_argument(
'-q', '--queue-size', type=int, default=128,
help="TODO")
help="Performance tuning value")
args_parser.add_argument(
'-a', '--autosave-interval', type=int, default=900,
help="TODO seconds")
help="Interval to which the database will save in seconds. "
"0 to disable.")
args_parser.add_argument(
'-4', '--ip4-cache', type=int, default=0,
help="TODO bytes max 512 MiB")
help="RAM cache for faster IPv4 lookup. "
"Maximum useful value: 512 MiB (536870912). "
"Warning: Depending on the rules, this might already "
"be a memory-heavy process, even without the cache.")
args = args_parser.parse_args()

recs_queue: multiprocessing.Queue = multiprocessing.Queue(
maxsize=args.queue_size)
maxsize=args.queue_size)

writer = Writer(recs_queue, autosave_interval=args.autosave_interval, ip4_cache=args.ip4_cache)
writer = Writer(recs_queue,
autosave_interval=args.autosave_interval,
ip4_cache=args.ip4_cache
)
writer.start()

parser = PARSERS[args.parser](args.input, recs_queue, args.block_size)


+ 5
- 3
feed_rules.py View File

@@ -7,22 +7,24 @@ import time

FUNCTION_MAP = {
'zone': database.Database.set_zone,
'ip4network': database.Database.set_ip4network,
'hostname': database.Database.set_hostname,
'asn': database.Database.set_asn,
'ip4network': database.Database.set_ip4network,
'ip4address': database.Database.set_ip4address,
}

if __name__ == '__main__':

# Parsing arguments
parser = argparse.ArgumentParser(
description="TODO")
description="Import base rules to the database")
parser.add_argument(
'type',
choices=FUNCTION_MAP.keys(),
help="Type of rule inputed")
parser.add_argument(
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
help="List of domains domains to block (with their subdomains)")
help="File with one rule per line")
parser.add_argument(
'-f', '--first-party', action='store_true',
help="The input only comes from verified first-party sources")


+ 1
- 1
import_rapid7.sh View File

@@ -12,7 +12,7 @@ function feed_rapid7_fdns { # dataset
curl -L "$link" | gunzip
}

function feed_rapid7_rdns { # dataset
function feed_rapid7_rdns {
dataset=$1
line=$(curl -s https://opendata.rapid7.com/sonar.rdns_v2/ | grep "href=\".\+-rdns.json.gz\"")
link="https://opendata.rapid7.com$(echo "$line" | cut -d'"' -f2)"


+ 4
- 4
validate_list.py View File

@@ -13,16 +13,16 @@ if __name__ == '__main__':

# Parsing arguments
parser = argparse.ArgumentParser(
description="Filter out invalid domain names.")
description="Filter out invalid domain name/ip addresses from a list.")
parser.add_argument(
'-i', '--input', type=argparse.FileType('r'), default=sys.stdin,
help="TODO")
help="Input file, one element per line")
parser.add_argument(
'-o', '--output', type=argparse.FileType('w'), default=sys.stdout,
help="TODO")
help="Output file, one element per line")
parser.add_argument(
'-d', '--domain', action='store_true',
help="Can be domain")
help="Can be domain name")
parser.add_argument(
'-4', '--ip4', action='store_true',
help="Can be IP4")


Loading…
Cancel
Save