From f3eedcba2260108d1e854ef1b0daca93f93b2eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Fri, 13 Dec 2019 13:54:00 +0100 Subject: [PATCH] Updated now based on timestamp Did I forget to add feed_asn.py a few commits ago? Oh well... --- database.py | 30 +++++++---------------------- feed_asn.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ feed_dns.py | 7 ++++--- feed_rules.py | 7 ++++++- 4 files changed, 69 insertions(+), 27 deletions(-) create mode 100755 feed_asn.py diff --git a/database.py b/database.py index d336eba..2a3e8df 100755 --- a/database.py +++ b/database.py @@ -98,13 +98,6 @@ class Database(): version) self.initialize() - updated = self.get_meta('updated') - if updated is None: - self.execute('SELECT max(updated) FROM rules') - data = self.cursor.fetchone() - updated, = data - self.updated = updated or 1 - def enter_step(self, name: str) -> None: now = time.perf_counter() try: @@ -168,20 +161,15 @@ class Database(): return mini, maxi # return Database.prepare_ip4address(net.network_address.exploded)[:net.prefixlen] - def expire(self) -> None: - self.enter_step('expire') - self.updated += 1 - self.set_meta('updated', self.updated) - def update_references(self) -> None: self.enter_step('update_refs') self.execute('UPDATE rules AS r SET refs=' '(SELECT count(*) FROM rules ' 'WHERE source=r.id)') - def prune(self) -> None: + def prune(self, before: int) -> None: self.enter_step('prune') - self.execute('DELETE FROM rules WHERE updated typing.Iterable[str]: @@ -264,6 +252,7 @@ class Database(): select_query: str, insert_query: str, prep: typing.Dict[str, DbValue], + updated: int, is_first_party: bool = False, source: int = None, ) -> None: @@ -287,9 +276,9 @@ class Database(): self.enter_step(f'set_{table}_select') self.execute(select_query, prep) - rules_prep = { + rules_prep: typing.Dict[str, DbValue] = { "source": source, - "updated": self.updated, + "updated": updated, "first_party": first_party, "level": level, } @@ -437,10 +426,7 @@ if __name__ == '__main__': help="Reconstruct the whole database") parser.add_argument( '-p', '--prune', action='store_true', - help="Remove old entries from database") - parser.add_argument( - '-e', '--expire', action='store_true', - help="Set the whole database as an old source") + help="Remove old (+6 months) entries from database") parser.add_argument( '-r', '--references', action='store_true', help="Update the reference count") @@ -451,9 +437,7 @@ if __name__ == '__main__': if args.initialize: DB.initialize() if args.prune: - DB.prune() - if args.expire: - DB.expire() + DB.prune(before=int(time.time()) - 60*60*24*31*6) if args.references and not args.prune: DB.update_references() diff --git a/feed_asn.py b/feed_asn.py new file mode 100755 index 0000000..a1343c0 --- /dev/null +++ b/feed_asn.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import database +import argparse +import requests +import typing +import ipaddress +import logging +import time + +IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network] + + +def get_ranges(asn: str) -> typing.Iterable[str]: + req = requests.get( + 'https://stat.ripe.net/data/as-routing-consistency/data.json', + params={'resource': asn} + ) + data = req.json() + for pref in data['data']['prefixes']: + yield pref['prefix'] + + +if __name__ == '__main__': + + log = logging.getLogger('feed_asn') + + # Parsing arguments + parser = argparse.ArgumentParser( + description="TODO") + args = parser.parse_args() + + DB = database.Database() + DBW = database.Database(write=True) + + for asn, entry in DB.list_asn(): + DB.enter_step('asn_get_ranges') + for prefix in get_ranges(asn): + parsed_prefix: IPNetwork = ipaddress.ip_network(prefix) + if parsed_prefix.version == 4: + DBW.set_ip4network( + prefix, + source=entry, + updated=int(time.time()) + ) + log.info('Added %s from %s (id=%s)', prefix, asn, entry) + elif parsed_prefix.version == 6: + log.warning('Unimplemented prefix version: %s', prefix) + else: + log.error('Unknown prefix version: %s', prefix) + + DB.close() diff --git a/feed_dns.py b/feed_dns.py index 2993d6d..e3cc02c 100755 --- a/feed_dns.py +++ b/feed_dns.py @@ -28,6 +28,7 @@ if __name__ == '__main__': # split = line.split(b'"') split = line.split('"') try: + updated = int(split[3]) name = split[7] dtype = split[11] value = split[15] @@ -43,13 +44,13 @@ if __name__ == '__main__': DB.enter_step('feed_switch') if dtype == 'a': for rule in DB.get_ip4(value): - DB.set_hostname(name, source=rule) + DB.set_hostname(name, source=rule, updated=updated) elif dtype == 'cname': for rule in DB.get_domain(value): - DB.set_hostname(name, source=rule) + DB.set_hostname(name, source=rule, updated=updated) elif dtype == 'ptr': for rule in DB.get_domain(value): - DB.set_ip4address(name, source=rule) + DB.set_ip4address(name, source=rule, updated=updated) DB.enter_step('iowait') except KeyboardInterrupt: log.warning("Interupted.") diff --git a/feed_rules.py b/feed_rules.py index 72888f5..715126e 100755 --- a/feed_rules.py +++ b/feed_rules.py @@ -3,6 +3,7 @@ import database import argparse import sys +import time FUNCTION_MAP = { 'zone': database.Database.set_zone, @@ -32,6 +33,10 @@ if __name__ == '__main__': fun = FUNCTION_MAP[args.type] for rule in args.input: - fun(DB, rule.strip(), is_first_party=args.first_party) + fun(DB, + rule.strip(), + is_first_party=args.first_party, + updated=int(time.time()), + ) DB.close()