|
|
@ -92,7 +92,9 @@ class Match(): |
|
|
|
|
|
|
|
|
|
|
|
class AsnNode(Match): |
|
|
|
pass |
|
|
|
def __init__(self) -> None: |
|
|
|
Match.__init__(self) |
|
|
|
self.name = '' |
|
|
|
|
|
|
|
|
|
|
|
class DomainTreeNode(): |
|
|
@ -111,8 +113,7 @@ class IpTreeNode(Match): |
|
|
|
|
|
|
|
Node = typing.Union[DomainTreeNode, IpTreeNode, AsnNode] |
|
|
|
MatchCallable = typing.Callable[[Path, |
|
|
|
Match, |
|
|
|
typing.Optional[typing.Any]], |
|
|
|
Match], |
|
|
|
typing.Any] |
|
|
|
|
|
|
|
|
|
|
@ -284,7 +285,6 @@ class Database(Profiler): |
|
|
|
|
|
|
|
def exec_each_asn(self, |
|
|
|
callback: MatchCallable, |
|
|
|
arg: typing.Any = None, |
|
|
|
) -> typing.Any: |
|
|
|
for asn in self.asns: |
|
|
|
match = self.asns[asn] |
|
|
@ -292,7 +292,6 @@ class Database(Profiler): |
|
|
|
c = callback( |
|
|
|
AsnPath(asn), |
|
|
|
match, |
|
|
|
arg |
|
|
|
) |
|
|
|
try: |
|
|
|
yield from c |
|
|
@ -301,7 +300,6 @@ class Database(Profiler): |
|
|
|
|
|
|
|
def exec_each_domain(self, |
|
|
|
callback: MatchCallable, |
|
|
|
arg: typing.Any = None, |
|
|
|
_dic: DomainTreeNode = None, |
|
|
|
_par: DomainPath = None, |
|
|
|
) -> typing.Any: |
|
|
@ -311,7 +309,6 @@ class Database(Profiler): |
|
|
|
c = callback( |
|
|
|
HostnamePath(_par.parts), |
|
|
|
_dic.match_hostname, |
|
|
|
arg |
|
|
|
) |
|
|
|
try: |
|
|
|
yield from c |
|
|
@ -321,7 +318,6 @@ class Database(Profiler): |
|
|
|
c = callback( |
|
|
|
ZonePath(_par.parts), |
|
|
|
_dic.match_zone, |
|
|
|
arg |
|
|
|
) |
|
|
|
try: |
|
|
|
yield from c |
|
|
@ -331,14 +327,12 @@ class Database(Profiler): |
|
|
|
dic = _dic.children[part] |
|
|
|
yield from self.exec_each_domain( |
|
|
|
callback, |
|
|
|
arg, |
|
|
|
_dic=dic, |
|
|
|
_par=DomainPath(_par.parts + [part]) |
|
|
|
) |
|
|
|
|
|
|
|
def exec_each_ip4(self, |
|
|
|
callback: MatchCallable, |
|
|
|
arg: typing.Any = None, |
|
|
|
_dic: IpTreeNode = None, |
|
|
|
_par: Ip4Path = None, |
|
|
|
) -> typing.Any: |
|
|
@ -348,7 +342,6 @@ class Database(Profiler): |
|
|
|
c = callback( |
|
|
|
_par, |
|
|
|
_dic, |
|
|
|
arg |
|
|
|
) |
|
|
|
try: |
|
|
|
yield from c |
|
|
@ -363,7 +356,6 @@ class Database(Profiler): |
|
|
|
assert addr0 == _par.value |
|
|
|
yield from self.exec_each_ip4( |
|
|
|
callback, |
|
|
|
arg, |
|
|
|
_dic=dic, |
|
|
|
_par=Ip4Path(addr0, pref) |
|
|
|
) |
|
|
@ -373,14 +365,12 @@ class Database(Profiler): |
|
|
|
addr1 = _par.value | (1 << (32-pref)) |
|
|
|
yield from self.exec_each_ip4( |
|
|
|
callback, |
|
|
|
arg, |
|
|
|
_dic=dic, |
|
|
|
_par=Ip4Path(addr1, pref) |
|
|
|
) |
|
|
|
|
|
|
|
def exec_each(self, |
|
|
|
callback: MatchCallable, |
|
|
|
arg: typing.Any = None, |
|
|
|
) -> typing.Any: |
|
|
|
yield from self.exec_each_domain(callback) |
|
|
|
yield from self.exec_each_ip4(callback) |
|
|
@ -390,19 +380,19 @@ class Database(Profiler): |
|
|
|
# Should be correctly calculated normally, |
|
|
|
# keeping this just in case |
|
|
|
def reset_references_cb(path: Path, |
|
|
|
match: Match, _: typing.Any |
|
|
|
match: Match |
|
|
|
) -> None: |
|
|
|
match.references = 0 |
|
|
|
for _ in self.exec_each(reset_references_cb, None): |
|
|
|
for _ in self.exec_each(reset_references_cb): |
|
|
|
pass |
|
|
|
|
|
|
|
def increment_references_cb(path: Path, |
|
|
|
match: Match, _: typing.Any |
|
|
|
match: Match |
|
|
|
) -> None: |
|
|
|
if match.source: |
|
|
|
source = self.get_match(match.source) |
|
|
|
source.references += 1 |
|
|
|
for _ in self.exec_each(increment_references_cb, None): |
|
|
|
for _ in self.exec_each(increment_references_cb): |
|
|
|
pass |
|
|
|
|
|
|
|
def prune(self, before: int, base_only: bool = False) -> None: |
|
|
@ -410,7 +400,10 @@ class Database(Profiler): |
|
|
|
|
|
|
|
def explain(self, path: Path) -> str: |
|
|
|
match = self.get_match(path) |
|
|
|
string = f'{path} #{match.references}' |
|
|
|
if isinstance(match, AsnNode): |
|
|
|
string = f'{path} ({match.name}) #{match.references}' |
|
|
|
else: |
|
|
|
string = f'{path} #{match.references}' |
|
|
|
if match.source: |
|
|
|
string += f' ← {self.explain(match.source)}' |
|
|
|
return string |
|
|
@ -421,7 +414,7 @@ class Database(Profiler): |
|
|
|
explain: bool = False, |
|
|
|
) -> typing.Iterable[str]: |
|
|
|
|
|
|
|
def export_cb(path: Path, match: Match, _: typing.Any |
|
|
|
def export_cb(path: Path, match: Match |
|
|
|
) -> typing.Iterable[str]: |
|
|
|
assert isinstance(path, DomainPath) |
|
|
|
if not isinstance(path, HostnamePath): |
|
|
@ -435,27 +428,49 @@ class Database(Profiler): |
|
|
|
else: |
|
|
|
yield self.unpack_domain(path) |
|
|
|
|
|
|
|
yield from self.exec_each_domain(export_cb, None) |
|
|
|
yield from self.exec_each_domain(export_cb) |
|
|
|
|
|
|
|
def list_rules(self, |
|
|
|
first_party_only: bool = False, |
|
|
|
) -> typing.Iterable[str]: |
|
|
|
|
|
|
|
def list_rules_cb(path: Path, match: Match, _: typing.Any |
|
|
|
def list_rules_cb(path: Path, match: Match |
|
|
|
) -> typing.Iterable[str]: |
|
|
|
if first_party_only and not match.first_party: |
|
|
|
return |
|
|
|
if isinstance(path, ZonePath) \ |
|
|
|
or (isinstance(path, Ip4Path) and path.prefixlen < 32): |
|
|
|
# if match.level == 0: |
|
|
|
# if match.level == 1: |
|
|
|
# It should be the latter condition but it is more |
|
|
|
# useful when using the former |
|
|
|
yield self.explain(path) |
|
|
|
|
|
|
|
yield from self.exec_each(list_rules_cb, None) |
|
|
|
yield from self.exec_each(list_rules_cb) |
|
|
|
|
|
|
|
def count_rules(self, |
|
|
|
def count_records(self, |
|
|
|
first_party_only: bool = False, |
|
|
|
rules_only: bool = False, |
|
|
|
) -> str: |
|
|
|
raise NotImplementedError |
|
|
|
memo: typing.Dict[str, int] = dict() |
|
|
|
|
|
|
|
def count_records_cb(path: Path, match: Match) -> None: |
|
|
|
if first_party_only and not match.first_party: |
|
|
|
return |
|
|
|
# if isinstance(path, ZonePath) \ |
|
|
|
# or (isinstance(path, Ip4Path) and path.prefixlen < 32): |
|
|
|
if rules_only and match.level > 1: |
|
|
|
return |
|
|
|
try: |
|
|
|
memo[path.__class__.__name__] += 1 |
|
|
|
except KeyError: |
|
|
|
memo[path.__class__.__name__] = 1 |
|
|
|
|
|
|
|
for _ in self.exec_each(count_records_cb): |
|
|
|
pass |
|
|
|
split: typing.List[str] = list() |
|
|
|
for key, value in sorted(memo.items(), key=lambda s: s[0]): |
|
|
|
split.append(f'{key[:-4]}: {value}') |
|
|
|
return ', '.join(split) |
|
|
|
|
|
|
|
def get_domain(self, domain_str: str) -> typing.Iterable[DomainPath]: |
|
|
|
self.enter_step('get_domain_pack') |
|
|
@ -486,7 +501,6 @@ class Database(Profiler): |
|
|
|
dic = self.ip4tree |
|
|
|
for i in range(31, 31-ip4.prefixlen, -1): |
|
|
|
bit = (ip4.value >> i) & 0b1 |
|
|
|
# TODO PERF copy value and slide once every loop |
|
|
|
if dic.active(): |
|
|
|
self.enter_step('get_ip4_yield') |
|
|
|
yield Ip4Path(ip4.value >> (i+1) << (i+1), 31-i) |
|
|
|