Updated now based on timestamp

Did I forget to add feed_asn.py a few commits ago?
Oh well...

database.py

@@ -98,13 +98,6 @@ class Database():
                     version)
             self.initialize()
 
-        updated = self.get_meta('updated')
-        if updated is None:
-            self.execute('SELECT max(updated) FROM rules')
-            data = self.cursor.fetchone()
-            updated, = data
-        self.updated = updated or 1
-
     def enter_step(self, name: str) -> None:
         now = time.perf_counter()
         try:
@@ -168,20 +161,15 @@ class Database():
         return mini, maxi
         # return Database.prepare_ip4address(net.network_address.exploded)[:net.prefixlen]
 
-    def expire(self) -> None:
-        self.enter_step('expire')
-        self.updated += 1
-        self.set_meta('updated', self.updated)
-
     def update_references(self) -> None:
         self.enter_step('update_refs')
         self.execute('UPDATE rules AS r SET refs='
                      '(SELECT count(*) FROM rules '
                      'WHERE source=r.id)')
 
-    def prune(self) -> None:
+    def prune(self, before: int) -> None:
         self.enter_step('prune')
-        self.execute('DELETE FROM rules WHERE updated<?', (self.updated,))
+        self.execute('DELETE FROM rules WHERE updated<?', (before,))
 
     def export(self, first_party_only: bool = False,
                end_chain_only: bool = False) -> typing.Iterable[str]:
@@ -264,6 +252,7 @@ class Database():
                      select_query: str,
                      insert_query: str,
                      prep: typing.Dict[str, DbValue],
+                     updated: int,
                      is_first_party: bool = False,
                      source: int = None,
                      ) -> None:
@@ -287,9 +276,9 @@ class Database():
         self.enter_step(f'set_{table}_select')
         self.execute(select_query, prep)
 
-        rules_prep = {
+        rules_prep: typing.Dict[str, DbValue] = {
             "source": source,
-            "updated": self.updated,
+            "updated": updated,
             "first_party": first_party,
             "level": level,
         }
@@ -437,10 +426,7 @@ if __name__ == '__main__':
         help="Reconstruct the whole database")
     parser.add_argument(
         '-p', '--prune', action='store_true',
-        help="Remove old entries from database")
+        help="Remove old (+6 months) entries from database")
-    parser.add_argument(
-        '-e', '--expire', action='store_true',
-        help="Set the whole database as an old source")
     parser.add_argument(
         '-r', '--references', action='store_true',
         help="Update the reference count")
@@ -451,9 +437,7 @@ if __name__ == '__main__':
     if args.initialize:
         DB.initialize()
     if args.prune:
-        DB.prune()
+        DB.prune(before=int(time.time()) - 60*60*24*31*6)
-    if args.expire:
-        DB.expire()
     if args.references and not args.prune:
         DB.update_references()
 

feed_asn.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import database
+import argparse
+import requests
+import typing
+import ipaddress
+import logging
+import time
+
+IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
+
+
+def get_ranges(asn: str) -> typing.Iterable[str]:
+    req = requests.get(
+        'https://stat.ripe.net/data/as-routing-consistency/data.json',
+        params={'resource': asn}
+    )
+    data = req.json()
+    for pref in data['data']['prefixes']:
+        yield pref['prefix']
+
+
+if __name__ == '__main__':
+
+    log = logging.getLogger('feed_asn')
+
+    # Parsing arguments
+    parser = argparse.ArgumentParser(
+        description="TODO")
+    args = parser.parse_args()
+
+    DB = database.Database()
+    DBW = database.Database(write=True)
+
+    for asn, entry in DB.list_asn():
+        DB.enter_step('asn_get_ranges')
+        for prefix in get_ranges(asn):
+            parsed_prefix: IPNetwork = ipaddress.ip_network(prefix)
+            if parsed_prefix.version == 4:
+                DBW.set_ip4network(
+                    prefix,
+                    source=entry,
+                    updated=int(time.time())
+                )
+                log.info('Added %s from %s (id=%s)', prefix, asn, entry)
+            elif parsed_prefix.version == 6:
+                log.warning('Unimplemented prefix version: %s', prefix)
+            else:
+                log.error('Unknown prefix version: %s', prefix)
+
+    DB.close()

feed_dns.py

@@ -28,6 +28,7 @@ if __name__ == '__main__':
             # split = line.split(b'"')
             split = line.split('"')
             try:
+                updated = int(split[3])
                 name = split[7]
                 dtype = split[11]
                 value = split[15]
@@ -43,13 +44,13 @@ if __name__ == '__main__':
             DB.enter_step('feed_switch')
             if dtype == 'a':
                 for rule in DB.get_ip4(value):
-                    DB.set_hostname(name, source=rule)
+                    DB.set_hostname(name, source=rule, updated=updated)
             elif dtype == 'cname':
                 for rule in DB.get_domain(value):
-                    DB.set_hostname(name, source=rule)
+                    DB.set_hostname(name, source=rule, updated=updated)
             elif dtype == 'ptr':
                 for rule in DB.get_domain(value):
-                    DB.set_ip4address(name, source=rule)
+                    DB.set_ip4address(name, source=rule, updated=updated)
             DB.enter_step('iowait')
     except KeyboardInterrupt:
         log.warning("Interupted.")

feed_rules.py

@@ -3,6 +3,7 @@
 import database
 import argparse
 import sys
+import time
 
 FUNCTION_MAP = {
     'zone': database.Database.set_zone,
@@ -32,6 +33,10 @@ if __name__ == '__main__':
     fun = FUNCTION_MAP[args.type]
 
     for rule in args.input:
-        fun(DB, rule.strip(), is_first_party=args.first_party)
+        fun(DB,
+            rule.strip(),
+            is_first_party=args.first_party,
+            updated=int(time.time()),
+            )
 
     DB.close()