|
|
@ -9,6 +9,8 @@ import time |
|
|
|
import logging |
|
|
|
import coloredlogs |
|
|
|
import pickle |
|
|
|
import numpy |
|
|
|
import math |
|
|
|
|
|
|
|
TLD_LIST: typing.Set[str] = set() |
|
|
|
|
|
|
@ -201,6 +203,33 @@ class Database(Profiler): |
|
|
|
Profiler.__init__(self) |
|
|
|
self.log = logging.getLogger('db') |
|
|
|
self.load() |
|
|
|
self.ip4cache_shift: int = 32 |
|
|
|
self.ip4cache = numpy.ones(1) |
|
|
|
|
|
|
|
def _set_ip4cache(self, path: Path, _: Match) -> None: |
|
|
|
assert isinstance(path, Ip4Path) |
|
|
|
self.enter_step('set_ip4cache') |
|
|
|
mini = path.value >> self.ip4cache_shift |
|
|
|
maxi = (path.value + 2**(32-path.prefixlen)) >> self.ip4cache_shift |
|
|
|
if mini == maxi: |
|
|
|
self.ip4cache[mini] = True |
|
|
|
else: |
|
|
|
self.ip4cache[mini:maxi] = True |
|
|
|
|
|
|
|
def fill_ip4cache(self, max_size: int = 512*1024**2) -> None: |
|
|
|
""" |
|
|
|
Size in bytes |
|
|
|
""" |
|
|
|
if max_size > 2**32/8: |
|
|
|
self.log.warning("Allocating more than 512 MiB of RAM for " |
|
|
|
"the Ip4 cache is not necessary.") |
|
|
|
max_cache_width = int(math.log2(max(1, max_size*8))) |
|
|
|
cache_width = min(2**32, max_cache_width) |
|
|
|
self.ip4cache_shift = 32-cache_width |
|
|
|
cache_size = 2**cache_width |
|
|
|
self.ip4cache = numpy.zeros(cache_size, dtype=numpy.bool) |
|
|
|
for _ in self.exec_each_ip4(self._set_ip4cache): |
|
|
|
pass |
|
|
|
|
|
|
|
@staticmethod |
|
|
|
def populate_tld_list() -> None: |
|
|
@ -404,8 +433,9 @@ class Database(Profiler): |
|
|
|
pref = _par.prefixlen + 1 |
|
|
|
dic = _dic.zero |
|
|
|
if dic: |
|
|
|
addr0 = _par.value & (0xFFFFFFFF ^ (1 << (32-pref))) |
|
|
|
assert addr0 == _par.value |
|
|
|
# addr0 = _par.value & (0xFFFFFFFF ^ (1 << (32-pref))) |
|
|
|
# assert addr0 == _par.value |
|
|
|
addr0 = _par.value |
|
|
|
yield from self.exec_each_ip4( |
|
|
|
callback, |
|
|
|
_dic=dic, |
|
|
@ -415,6 +445,7 @@ class Database(Profiler): |
|
|
|
dic = _dic.one |
|
|
|
if dic: |
|
|
|
addr1 = _par.value | (1 << (32-pref)) |
|
|
|
# assert addr1 != _par.value |
|
|
|
yield from self.exec_each_ip4( |
|
|
|
callback, |
|
|
|
_dic=dic, |
|
|
@ -548,6 +579,9 @@ class Database(Profiler): |
|
|
|
def get_ip4(self, ip4_str: str) -> typing.Iterable[Path]: |
|
|
|
self.enter_step('get_ip4_pack') |
|
|
|
ip4 = self.pack_ip4address(ip4_str) |
|
|
|
self.enter_step('get_ip4_cache') |
|
|
|
if not self.ip4cache[ip4.value >> self.ip4cache_shift]: |
|
|
|
return |
|
|
|
self.enter_step('get_ip4_brws') |
|
|
|
dic = self.ip4tree |
|
|
|
for i in range(31, 31-ip4.prefixlen, -1): |
|
|
@ -680,6 +714,7 @@ class Database(Profiler): |
|
|
|
source_match=source_match, |
|
|
|
dupplicate=dupplicate, |
|
|
|
) |
|
|
|
self._set_ip4cache(ip4, dic) |
|
|
|
|
|
|
|
def set_ip4address(self, |
|
|
|
ip4address_str: str, |
|
|
|