geoffrey/eulaurarien
1
0
Fork 0
Code Issues 7 Pull requests Releases 1 Wiki Activity

Updated now based on timestamp

Browse source 
Did I forget to add feed_asn.py a few commits ago?
Oh well...
This commit is contained in:
Geoffrey Frogeye 2019-12-13 13:54:00 +01:00
parent 8d94b80fd0
commit f3eedcba22
Signed by: geoffrey
GPG key ID: D8A7ECA00A8CD3DD
4 changed files with 69 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

30
database.py
View file

@ -98,13 +98,6 @@ class Database():
version)
self.initialize()
updated = self.get_meta('updated')
if updated is None:
self.execute('SELECT max(updated) FROM rules')
data = self.cursor.fetchone()
updated, = data
self.updated = updated or 1
def enter_step(self, name: str) -> None:
now = time.perf_counter()
try:
@ -168,20 +161,15 @@ class Database():
return mini, maxi
# return Database.prepare_ip4address(net.network_address.exploded)[:net.prefixlen]
def expire(self) -> None:
self.enter_step('expire')
self.updated += 1
self.set_meta('updated', self.updated)
def update_references(self) -> None:
self.enter_step('update_refs')
self.execute('UPDATE rules AS r SET refs='
'(SELECT count(*) FROM rules '
'WHERE source=r.id)')
def prune(self) -> None:
def prune(self, before: int) -> None:
self.enter_step('prune')
self.execute('DELETE FROM rules WHERE updated<?', (self.updated,))
self.execute('DELETE FROM rules WHERE updated<?', (before,))
def export(self, first_party_only: bool = False,
end_chain_only: bool = False) -> typing.Iterable[str]:
@ -264,6 +252,7 @@ class Database():
select_query: str,
insert_query: str,
prep: typing.Dict[str, DbValue],
updated: int,
is_first_party: bool = False,
source: int = None,
) -> None:
@ -287,9 +276,9 @@ class Database():
self.enter_step(f'set_{table}_select')
self.execute(select_query, prep)
rules_prep = {
rules_prep: typing.Dict[str, DbValue] = {
"source": source,
"updated": self.updated,
"updated": updated,
"first_party": first_party,
"level": level,
}
@ -437,10 +426,7 @@ if __name__ == '__main__':
help="Reconstruct the whole database")
parser.add_argument(
'-p', '--prune', action='store_true',
help="Remove old entries from database")
parser.add_argument(
'-e', '--expire', action='store_true',
help="Set the whole database as an old source")
help="Remove old (+6 months) entries from database")
parser.add_argument(
'-r', '--references', action='store_true',
help="Update the reference count")
@ -451,9 +437,7 @@ if __name__ == '__main__':
if args.initialize:
DB.initialize()
if args.prune:
DB.prune()
if args.expire:
DB.expire()
DB.prune(before=int(time.time()) - 60*60*24*31*6)
if args.references and not args.prune:
DB.update_references()