Updated now based on timestamp

Did I forget to add feed_asn.py a few commits ago?
Oh well...
This commit is contained in:
Geoffrey Frogeye 2019-12-13 13:54:00 +01:00
parent 8d94b80fd0
commit f3eedcba22
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
4 changed files with 69 additions and 27 deletions

View file

@ -98,13 +98,6 @@ class Database():
version)
self.initialize()
updated = self.get_meta('updated')
if updated is None:
self.execute('SELECT max(updated) FROM rules')
data = self.cursor.fetchone()
updated, = data
self.updated = updated or 1
def enter_step(self, name: str) -> None:
now = time.perf_counter()
try:
@ -168,20 +161,15 @@ class Database():
return mini, maxi
# return Database.prepare_ip4address(net.network_address.exploded)[:net.prefixlen]
def expire(self) -> None:
self.enter_step('expire')
self.updated += 1
self.set_meta('updated', self.updated)
def update_references(self) -> None:
self.enter_step('update_refs')
self.execute('UPDATE rules AS r SET refs='
'(SELECT count(*) FROM rules '
'WHERE source=r.id)')
def prune(self) -> None:
def prune(self, before: int) -> None:
self.enter_step('prune')
self.execute('DELETE FROM rules WHERE updated<?', (self.updated,))
self.execute('DELETE FROM rules WHERE updated<?', (before,))
def export(self, first_party_only: bool = False,
end_chain_only: bool = False) -> typing.Iterable[str]:
@ -264,6 +252,7 @@ class Database():
select_query: str,
insert_query: str,
prep: typing.Dict[str, DbValue],
updated: int,
is_first_party: bool = False,
source: int = None,
) -> None:
@ -287,9 +276,9 @@ class Database():
self.enter_step(f'set_{table}_select')
self.execute(select_query, prep)
rules_prep = {
rules_prep: typing.Dict[str, DbValue] = {
"source": source,
"updated": self.updated,
"updated": updated,
"first_party": first_party,
"level": level,
}
@ -437,10 +426,7 @@ if __name__ == '__main__':
help="Reconstruct the whole database")
parser.add_argument(
'-p', '--prune', action='store_true',
help="Remove old entries from database")
parser.add_argument(
'-e', '--expire', action='store_true',
help="Set the whole database as an old source")
help="Remove old (+6 months) entries from database")
parser.add_argument(
'-r', '--references', action='store_true',
help="Update the reference count")
@ -451,9 +437,7 @@ if __name__ == '__main__':
if args.initialize:
DB.initialize()
if args.prune:
DB.prune()
if args.expire:
DB.expire()
DB.prune(before=int(time.time()) - 60*60*24*31*6)
if args.references and not args.prune:
DB.update_references()

52
feed_asn.py Executable file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python3
import database
import argparse
import requests
import typing
import ipaddress
import logging
import time
IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
def get_ranges(asn: str) -> typing.Iterable[str]:
req = requests.get(
'https://stat.ripe.net/data/as-routing-consistency/data.json',
params={'resource': asn}
)
data = req.json()
for pref in data['data']['prefixes']:
yield pref['prefix']
if __name__ == '__main__':
log = logging.getLogger('feed_asn')
# Parsing arguments
parser = argparse.ArgumentParser(
description="TODO")
args = parser.parse_args()
DB = database.Database()
DBW = database.Database(write=True)
for asn, entry in DB.list_asn():
DB.enter_step('asn_get_ranges')
for prefix in get_ranges(asn):
parsed_prefix: IPNetwork = ipaddress.ip_network(prefix)
if parsed_prefix.version == 4:
DBW.set_ip4network(
prefix,
source=entry,
updated=int(time.time())
)
log.info('Added %s from %s (id=%s)', prefix, asn, entry)
elif parsed_prefix.version == 6:
log.warning('Unimplemented prefix version: %s', prefix)
else:
log.error('Unknown prefix version: %s', prefix)
DB.close()

View file

@ -28,6 +28,7 @@ if __name__ == '__main__':
# split = line.split(b'"')
split = line.split('"')
try:
updated = int(split[3])
name = split[7]
dtype = split[11]
value = split[15]
@ -43,13 +44,13 @@ if __name__ == '__main__':
DB.enter_step('feed_switch')
if dtype == 'a':
for rule in DB.get_ip4(value):
DB.set_hostname(name, source=rule)
DB.set_hostname(name, source=rule, updated=updated)
elif dtype == 'cname':
for rule in DB.get_domain(value):
DB.set_hostname(name, source=rule)
DB.set_hostname(name, source=rule, updated=updated)
elif dtype == 'ptr':
for rule in DB.get_domain(value):
DB.set_ip4address(name, source=rule)
DB.set_ip4address(name, source=rule, updated=updated)
DB.enter_step('iowait')
except KeyboardInterrupt:
log.warning("Interupted.")

View file

@ -3,6 +3,7 @@
import database
import argparse
import sys
import time
FUNCTION_MAP = {
'zone': database.Database.set_zone,
@ -32,6 +33,10 @@ if __name__ == '__main__':
fun = FUNCTION_MAP[args.type]
for rule in args.input:
fun(DB, rule.strip(), is_first_party=args.first_party)
fun(DB,
rule.strip(),
is_first_party=args.first_party,
updated=int(time.time()),
)
DB.close()