geoffrey/eulaurarien
1
0
Fork 0
Code Issues 7 Pull requests Releases 1 Wiki Activity

Added intermediate representation for DNS datasets

Browse source 
It's just CSV.
The DNS from the datasets are not ordered consistently,
so we need to parse it completly.
It seems that converting to an IR before sending data to ./feed_dns.py
through a pipe is faster than decoding the JSON in ./feed_dns.py.
This will also reduce the storage of the resolved subdomains by
about 15% (compressed).
This commit is contained in:
Geoffrey Frogeye 2019-12-13 21:59:35 +01:00
parent 269b8278b5
commit 5023b85d7c
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
5 changed files with 68 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

48
feed_dns.py
View file

@ -4,6 +4,8 @@ import database
import argparse
import sys
import logging
import csv
import json
if __name__ == '__main__':
@ -21,39 +23,39 @@ if __name__ == '__main__':
try:
DB.enter_step('iowait')
# line: bytes
line: str
for line in args.input:
DB.enter_step('feed_json_parse')
# split = line.split(b'"')
split = line.split('"')
try:
updated = int(split[3])
name = split[7]
dtype = split[11]
value = split[15]
except IndexError:
log.error("Invalid JSON: %s", line)
continue
# DB.enter_step('feed_json_assert')
for row in csv.reader(args.input):
# for line in args.input:
DB.enter_step('feed_csv_parse')
dtype, timestamp, name, value = row
# DB.enter_step('feed_json_parse')
# data = json.loads(line)
# assert dtype == data['type']
# assert name == data['name']
# assert value == data['value']
# dtype = data['type'][0]
# # timestamp = data['timestamp']
# name = data['name']
# value = data['value']
DB.enter_step('feed_switch')
if dtype == 'a':
for rule in DB.get_ip4(value):
if not list(DB.get_domain_in_zone(name)):
DB.set_hostname(name, source=rule, updated=updated)
elif dtype == 'cname':
DB.set_hostname(name, source=rule,
updated=int(timestamp))
# updated=int(data['timestamp']))
elif dtype == 'c':
for rule in DB.get_domain(value):
if not list(DB.get_domain_in_zone(name)):
DB.set_hostname(name, source=rule, updated=updated)
elif dtype == 'ptr':
DB.set_hostname(name, source=rule,
updated=int(timestamp))
# updated=int(data['timestamp']))
elif dtype == 'p':
for rule in DB.get_domain(value):
if not list(DB.get_ip4_in_network(name)):
DB.set_ip4address(name, source=rule, updated=updated)
DB.set_ip4address(name, source=rule,
updated=int(timestamp))
# updated=int(data['timestamp']))
else:
raise NotImplementedError(f'Type: {dtype}')
DB.enter_step('iowait')
except KeyboardInterrupt:
log.warning("Interupted.")