Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

799 lines
25 KiB

4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
  1. #!/usr/bin/env python3
  2. """
  3. Utility functions to interact with the database.
  4. """
  5. import typing
  6. import time
  7. import logging
  8. import coloredlogs
  9. import pickle
  10. import numpy
  11. import math
  12. import os
  13. TLD_LIST: typing.Set[str] = set()
  14. coloredlogs.install(level="DEBUG", fmt="%(asctime)s %(name)s %(levelname)s %(message)s")
  15. Asn = int
  16. Timestamp = int
  17. Level = int
  18. class Path:
  19. pass
  20. class RulePath(Path):
  21. def __str__(self) -> str:
  22. return "(rule)"
  23. class RuleFirstPath(RulePath):
  24. def __str__(self) -> str:
  25. return "(first-party rule)"
  26. class RuleMultiPath(RulePath):
  27. def __str__(self) -> str:
  28. return "(multi-party rule)"
  29. class DomainPath(Path):
  30. def __init__(self, parts: typing.List[str]):
  31. self.parts = parts
  32. def __str__(self) -> str:
  33. return "?." + Database.unpack_domain(self)
  34. class HostnamePath(DomainPath):
  35. def __str__(self) -> str:
  36. return Database.unpack_domain(self)
  37. class ZonePath(DomainPath):
  38. def __str__(self) -> str:
  39. return "*." + Database.unpack_domain(self)
  40. class AsnPath(Path):
  41. def __init__(self, asn: Asn):
  42. self.asn = asn
  43. def __str__(self) -> str:
  44. return Database.unpack_asn(self)
  45. class Ip4Path(Path):
  46. def __init__(self, value: int, prefixlen: int):
  47. self.value = value
  48. self.prefixlen = prefixlen
  49. def __str__(self) -> str:
  50. return Database.unpack_ip4network(self)
  51. class Match:
  52. def __init__(self) -> None:
  53. self.source: typing.Optional[Path] = None
  54. self.updated: int = 0
  55. self.dupplicate: bool = False
  56. # Cache
  57. self.level: int = 0
  58. self.first_party: bool = False
  59. self.references: int = 0
  60. def active(self, first_party: bool = None) -> bool:
  61. if self.updated == 0 or (first_party and not self.first_party):
  62. return False
  63. return True
  64. def disable(self) -> None:
  65. self.updated = 0
  66. class AsnNode(Match):
  67. def __init__(self) -> None:
  68. Match.__init__(self)
  69. self.name = ""
  70. class DomainTreeNode:
  71. def __init__(self) -> None:
  72. self.children: typing.Dict[str, DomainTreeNode] = dict()
  73. self.match_zone = Match()
  74. self.match_hostname = Match()
  75. class IpTreeNode(Match):
  76. def __init__(self) -> None:
  77. Match.__init__(self)
  78. self.zero: typing.Optional[IpTreeNode] = None
  79. self.one: typing.Optional[IpTreeNode] = None
  80. Node = typing.Union[DomainTreeNode, IpTreeNode, AsnNode]
  81. MatchCallable = typing.Callable[[Path, Match], typing.Any]
  82. class Profiler:
  83. def __init__(self) -> None:
  84. do_profile = int(os.environ.get("PROFILE", "0"))
  85. if do_profile:
  86. self.log = logging.getLogger("profiler")
  87. self.time_last = time.perf_counter()
  88. self.time_step = "init"
  89. self.time_dict: typing.Dict[str, float] = dict()
  90. self.step_dict: typing.Dict[str, int] = dict()
  91. self.enter_step = self.enter_step_real
  92. self.profile = self.profile_real
  93. else:
  94. self.enter_step = self.enter_step_dummy
  95. self.profile = self.profile_dummy
  96. def enter_step_dummy(self, name: str) -> None:
  97. return
  98. def enter_step_real(self, name: str) -> None:
  99. now = time.perf_counter()
  100. try:
  101. self.time_dict[self.time_step] += now - self.time_last
  102. self.step_dict[self.time_step] += int(name != self.time_step)
  103. except KeyError:
  104. self.time_dict[self.time_step] = now - self.time_last
  105. self.step_dict[self.time_step] = 1
  106. self.time_step = name
  107. self.time_last = time.perf_counter()
  108. def profile_dummy(self) -> None:
  109. return
  110. def profile_real(self) -> None:
  111. self.enter_step("profile")
  112. total = sum(self.time_dict.values())
  113. for key, secs in sorted(self.time_dict.items(), key=lambda t: t[1]):
  114. times = self.step_dict[key]
  115. self.log.debug(
  116. f"{key:<20}: {times:9d} × {secs/times:5.3e} "
  117. f"= {secs:9.2f} s ({secs/total:7.2%}) "
  118. )
  119. self.log.debug(
  120. f"{'total':<20}: " f"{total:9.2f} s ({1:7.2%})"
  121. )
  122. class Database(Profiler):
  123. VERSION = 18
  124. PATH = "blocking.p"
  125. def initialize(self) -> None:
  126. self.log.warning("Creating database version: %d ", Database.VERSION)
  127. # Dummy match objects that everything refer to
  128. self.rules: typing.List[Match] = list()
  129. for first_party in (False, True):
  130. m = Match()
  131. m.updated = 1
  132. m.level = 0
  133. m.first_party = first_party
  134. self.rules.append(m)
  135. self.domtree = DomainTreeNode()
  136. self.asns: typing.Dict[Asn, AsnNode] = dict()
  137. self.ip4tree = IpTreeNode()
  138. def load(self) -> None:
  139. self.enter_step("load")
  140. try:
  141. with open(self.PATH, "rb") as db_fdsec:
  142. version, data = pickle.load(db_fdsec)
  143. if version == Database.VERSION:
  144. self.rules, self.domtree, self.asns, self.ip4tree = data
  145. return
  146. self.log.warning(
  147. "Outdated database version found: %d, " "it will be rebuilt.",
  148. version,
  149. )
  150. except (TypeError, AttributeError, EOFError):
  151. self.log.error(
  152. "Corrupt (or heavily outdated) database found, " "it will be rebuilt."
  153. )
  154. except FileNotFoundError:
  155. pass
  156. self.initialize()
  157. def save(self) -> None:
  158. self.enter_step("save")
  159. with open(self.PATH, "wb") as db_fdsec:
  160. data = self.rules, self.domtree, self.asns, self.ip4tree
  161. pickle.dump((self.VERSION, data), db_fdsec)
  162. self.profile()
  163. def __init__(self) -> None:
  164. Profiler.__init__(self)
  165. self.log = logging.getLogger("db")
  166. self.load()
  167. self.ip4cache_shift: int = 32
  168. self.ip4cache = numpy.ones(1)
  169. def _set_ip4cache(self, path: Path, _: Match) -> None:
  170. assert isinstance(path, Ip4Path)
  171. self.enter_step("set_ip4cache")
  172. mini = path.value >> self.ip4cache_shift
  173. maxi = (path.value + 2 ** (32 - path.prefixlen)) >> self.ip4cache_shift
  174. if mini == maxi:
  175. self.ip4cache[mini] = True
  176. else:
  177. self.ip4cache[mini:maxi] = True
  178. def fill_ip4cache(self, max_size: int = 512 * 1024 ** 2) -> None:
  179. """
  180. Size in bytes
  181. """
  182. if max_size > 2 ** 32 / 8:
  183. self.log.warning(
  184. "Allocating more than 512 MiB of RAM for "
  185. "the Ip4 cache is not necessary."
  186. )
  187. max_cache_width = int(math.log2(max(1, max_size * 8)))
  188. allocated = False
  189. cache_width = min(32, max_cache_width)
  190. while not allocated:
  191. cache_size = 2 ** cache_width
  192. try:
  193. self.ip4cache = numpy.zeros(cache_size, dtype=bool)
  194. except MemoryError:
  195. self.log.exception("Could not allocate cache. Retrying a smaller one.")
  196. cache_width -= 1
  197. continue
  198. allocated = True
  199. self.ip4cache_shift = 32 - cache_width
  200. for _ in self.exec_each_ip4(self._set_ip4cache):
  201. pass
  202. @staticmethod
  203. def populate_tld_list() -> None:
  204. with open("temp/all_tld.list", "r") as tld_fdesc:
  205. for tld in tld_fdesc:
  206. tld = tld.strip()
  207. TLD_LIST.add(tld)
  208. @staticmethod
  209. def validate_domain(path: str) -> bool:
  210. if len(path) > 255:
  211. return False
  212. splits = path.split(".")
  213. if not TLD_LIST:
  214. Database.populate_tld_list()
  215. if splits[-1] not in TLD_LIST:
  216. return False
  217. for split in splits:
  218. if not 1 <= len(split) <= 63:
  219. return False
  220. return True
  221. @staticmethod
  222. def pack_domain(domain: str) -> DomainPath:
  223. return DomainPath(domain.split(".")[::-1])
  224. @staticmethod
  225. def unpack_domain(domain: DomainPath) -> str:
  226. return ".".join(domain.parts[::-1])
  227. @staticmethod
  228. def pack_asn(asn: str) -> AsnPath:
  229. asn = asn.upper()
  230. if asn.startswith("AS"):
  231. asn = asn[2:]
  232. return AsnPath(int(asn))
  233. @staticmethod
  234. def unpack_asn(asn: AsnPath) -> str:
  235. return f"AS{asn.asn}"
  236. @staticmethod
  237. def validate_ip4address(path: str) -> bool:
  238. splits = path.split(".")
  239. if len(splits) != 4:
  240. return False
  241. for split in splits:
  242. try:
  243. if not 0 <= int(split) <= 255:
  244. return False
  245. except ValueError:
  246. return False
  247. return True
  248. @staticmethod
  249. def pack_ip4address_low(address: str) -> int:
  250. addr = 0
  251. for split in address.split("."):
  252. octet = int(split)
  253. addr = (addr << 8) + octet
  254. return addr
  255. @staticmethod
  256. def pack_ip4address(address: str) -> Ip4Path:
  257. return Ip4Path(Database.pack_ip4address_low(address), 32)
  258. @staticmethod
  259. def unpack_ip4address(address: Ip4Path) -> str:
  260. addr = address.value
  261. assert address.prefixlen == 32
  262. octets: typing.List[int] = list()
  263. octets = [0] * 4
  264. for o in reversed(range(4)):
  265. octets[o] = addr & 0xFF
  266. addr >>= 8
  267. return ".".join(map(str, octets))
  268. @staticmethod
  269. def validate_ip4network(path: str) -> bool:
  270. # A bit generous but ok for our usage
  271. splits = path.split("/")
  272. if len(splits) != 2:
  273. return False
  274. if not Database.validate_ip4address(splits[0]):
  275. return False
  276. try:
  277. if not 0 <= int(splits[1]) <= 32:
  278. return False
  279. except ValueError:
  280. return False
  281. return True
  282. @staticmethod
  283. def pack_ip4network(network: str) -> Ip4Path:
  284. address, prefixlen_str = network.split("/")
  285. prefixlen = int(prefixlen_str)
  286. addr = Database.pack_ip4address(address)
  287. addr.prefixlen = prefixlen
  288. return addr
  289. @staticmethod
  290. def unpack_ip4network(network: Ip4Path) -> str:
  291. addr = network.value
  292. octets: typing.List[int] = list()
  293. octets = [0] * 4
  294. for o in reversed(range(4)):
  295. octets[o] = addr & 0xFF
  296. addr >>= 8
  297. return ".".join(map(str, octets)) + "/" + str(network.prefixlen)
  298. def get_match(self, path: Path) -> Match:
  299. if isinstance(path, RuleMultiPath):
  300. return self.rules[0]
  301. elif isinstance(path, RuleFirstPath):
  302. return self.rules[1]
  303. elif isinstance(path, AsnPath):
  304. return self.asns[path.asn]
  305. elif isinstance(path, DomainPath):
  306. dicd = self.domtree
  307. for part in path.parts:
  308. dicd = dicd.children[part]
  309. if isinstance(path, HostnamePath):
  310. return dicd.match_hostname
  311. elif isinstance(path, ZonePath):
  312. return dicd.match_zone
  313. else:
  314. raise ValueError
  315. elif isinstance(path, Ip4Path):
  316. dici = self.ip4tree
  317. for i in range(31, 31 - path.prefixlen, -1):
  318. bit = (path.value >> i) & 0b1
  319. dici_next = dici.one if bit else dici.zero
  320. if not dici_next:
  321. raise IndexError
  322. dici = dici_next
  323. return dici
  324. else:
  325. raise ValueError
  326. def exec_each_asn(
  327. self,
  328. callback: MatchCallable,
  329. ) -> typing.Any:
  330. for asn in self.asns:
  331. match = self.asns[asn]
  332. if match.active():
  333. c = callback(
  334. AsnPath(asn),
  335. match,
  336. )
  337. try:
  338. yield from c
  339. except TypeError: # not iterable
  340. pass
  341. def exec_each_domain(
  342. self,
  343. callback: MatchCallable,
  344. _dic: DomainTreeNode = None,
  345. _par: DomainPath = None,
  346. ) -> typing.Any:
  347. _dic = _dic or self.domtree
  348. _par = _par or DomainPath([])
  349. if _dic.match_hostname.active():
  350. c = callback(
  351. HostnamePath(_par.parts),
  352. _dic.match_hostname,
  353. )
  354. try:
  355. yield from c
  356. except TypeError: # not iterable
  357. pass
  358. if _dic.match_zone.active():
  359. c = callback(
  360. ZonePath(_par.parts),
  361. _dic.match_zone,
  362. )
  363. try:
  364. yield from c
  365. except TypeError: # not iterable
  366. pass
  367. for part in _dic.children:
  368. dic = _dic.children[part]
  369. yield from self.exec_each_domain(
  370. callback, _dic=dic, _par=DomainPath(_par.parts + [part])
  371. )
  372. def exec_each_ip4(
  373. self,
  374. callback: MatchCallable,
  375. _dic: IpTreeNode = None,
  376. _par: Ip4Path = None,
  377. ) -> typing.Any:
  378. _dic = _dic or self.ip4tree
  379. _par = _par or Ip4Path(0, 0)
  380. if _dic.active():
  381. c = callback(
  382. _par,
  383. _dic,
  384. )
  385. try:
  386. yield from c
  387. except TypeError: # not iterable
  388. pass
  389. # 0
  390. pref = _par.prefixlen + 1
  391. dic = _dic.zero
  392. if dic:
  393. # addr0 = _par.value & (0xFFFFFFFF ^ (1 << (32-pref)))
  394. # assert addr0 == _par.value
  395. addr0 = _par.value
  396. yield from self.exec_each_ip4(callback, _dic=dic, _par=Ip4Path(addr0, pref))
  397. # 1
  398. dic = _dic.one
  399. if dic:
  400. addr1 = _par.value | (1 << (32 - pref))
  401. # assert addr1 != _par.value
  402. yield from self.exec_each_ip4(callback, _dic=dic, _par=Ip4Path(addr1, pref))
  403. def exec_each(
  404. self,
  405. callback: MatchCallable,
  406. ) -> typing.Any:
  407. yield from self.exec_each_domain(callback)
  408. yield from self.exec_each_ip4(callback)
  409. yield from self.exec_each_asn(callback)
  410. def update_references(self) -> None:
  411. # Should be correctly calculated normally,
  412. # keeping this just in case
  413. def reset_references_cb(path: Path, match: Match) -> None:
  414. match.references = 0
  415. for _ in self.exec_each(reset_references_cb):
  416. pass
  417. def increment_references_cb(path: Path, match: Match) -> None:
  418. if match.source:
  419. source = self.get_match(match.source)
  420. source.references += 1
  421. for _ in self.exec_each(increment_references_cb):
  422. pass
  423. def _clean_deps(self) -> None:
  424. # Disable the matches that depends on the targeted
  425. # matches until all disabled matches reference count = 0
  426. did_something = True
  427. def clean_deps_cb(path: Path, match: Match) -> None:
  428. nonlocal did_something
  429. if not match.source:
  430. return
  431. source = self.get_match(match.source)
  432. if not source.active():
  433. self._unset_match(match)
  434. elif match.first_party > source.first_party:
  435. match.first_party = source.first_party
  436. else:
  437. return
  438. did_something = True
  439. while did_something:
  440. did_something = False
  441. self.enter_step("pass_clean_deps")
  442. for _ in self.exec_each(clean_deps_cb):
  443. pass
  444. def prune(self, before: int, base_only: bool = False) -> None:
  445. # Disable the matches targeted
  446. def prune_cb(path: Path, match: Match) -> None:
  447. if base_only and match.level > 1:
  448. return
  449. if match.updated > before:
  450. return
  451. self._unset_match(match)
  452. self.log.debug("Print: disabled %s", path)
  453. self.enter_step("pass_prune")
  454. for _ in self.exec_each(prune_cb):
  455. pass
  456. self._clean_deps()
  457. # Remove branches with no match
  458. # TODO
  459. def explain(self, path: Path) -> str:
  460. match = self.get_match(path)
  461. string = str(path)
  462. if isinstance(match, AsnNode):
  463. string += f" ({match.name})"
  464. party_char = "F" if match.first_party else "M"
  465. dup_char = "D" if match.dupplicate else "_"
  466. string += f" {match.level}{party_char}{dup_char}{match.references}"
  467. if match.source:
  468. string += f" ← {self.explain(match.source)}"
  469. return string
  470. def list_records(
  471. self,
  472. first_party_only: bool = False,
  473. end_chain_only: bool = False,
  474. no_dupplicates: bool = False,
  475. rules_only: bool = False,
  476. hostnames_only: bool = False,
  477. explain: bool = False,
  478. ) -> typing.Iterable[str]:
  479. def export_cb(path: Path, match: Match) -> typing.Iterable[str]:
  480. if first_party_only and not match.first_party:
  481. return
  482. if end_chain_only and match.references > 0:
  483. return
  484. if no_dupplicates and match.dupplicate:
  485. return
  486. if rules_only and match.level > 1:
  487. return
  488. if hostnames_only and not isinstance(path, HostnamePath):
  489. return
  490. if explain:
  491. yield self.explain(path)
  492. else:
  493. yield str(path)
  494. yield from self.exec_each(export_cb)
  495. def count_records(
  496. self,
  497. first_party_only: bool = False,
  498. end_chain_only: bool = False,
  499. no_dupplicates: bool = False,
  500. rules_only: bool = False,
  501. hostnames_only: bool = False,
  502. ) -> str:
  503. memo: typing.Dict[str, int] = dict()
  504. def count_records_cb(path: Path, match: Match) -> None:
  505. if first_party_only and not match.first_party:
  506. return
  507. if end_chain_only and match.references > 0:
  508. return
  509. if no_dupplicates and match.dupplicate:
  510. return
  511. if rules_only and match.level > 1:
  512. return
  513. if hostnames_only and not isinstance(path, HostnamePath):
  514. return
  515. try:
  516. memo[path.__class__.__name__] += 1
  517. except KeyError:
  518. memo[path.__class__.__name__] = 1
  519. for _ in self.exec_each(count_records_cb):
  520. pass
  521. split: typing.List[str] = list()
  522. for key, value in sorted(memo.items(), key=lambda s: s[0]):
  523. split.append(f"{key[:-4].lower()}s: {value}")
  524. return ", ".join(split)
  525. def get_domain(self, domain_str: str) -> typing.Iterable[DomainPath]:
  526. self.enter_step("get_domain_pack")
  527. domain = self.pack_domain(domain_str)
  528. self.enter_step("get_domain_brws")
  529. dic = self.domtree
  530. depth = 0
  531. for part in domain.parts:
  532. if dic.match_zone.active():
  533. self.enter_step("get_domain_yield")
  534. yield ZonePath(domain.parts[:depth])
  535. self.enter_step("get_domain_brws")
  536. if part not in dic.children:
  537. return
  538. dic = dic.children[part]
  539. depth += 1
  540. if dic.match_zone.active():
  541. self.enter_step("get_domain_yield")
  542. yield ZonePath(domain.parts)
  543. if dic.match_hostname.active():
  544. self.enter_step("get_domain_yield")
  545. yield HostnamePath(domain.parts)
  546. def get_ip4(self, ip4_str: str) -> typing.Iterable[Path]:
  547. self.enter_step("get_ip4_pack")
  548. ip4val = self.pack_ip4address_low(ip4_str)
  549. self.enter_step("get_ip4_cache")
  550. if not self.ip4cache[ip4val >> self.ip4cache_shift]:
  551. return
  552. self.enter_step("get_ip4_brws")
  553. dic = self.ip4tree
  554. for i in range(31, -1, -1):
  555. bit = (ip4val >> i) & 0b1
  556. if dic.active():
  557. self.enter_step("get_ip4_yield")
  558. yield Ip4Path(ip4val >> (i + 1) << (i + 1), 31 - i)
  559. self.enter_step("get_ip4_brws")
  560. next_dic = dic.one if bit else dic.zero
  561. if next_dic is None:
  562. return
  563. dic = next_dic
  564. if dic.active():
  565. self.enter_step("get_ip4_yield")
  566. yield Ip4Path(ip4val, 32)
  567. def _unset_match(
  568. self,
  569. match: Match,
  570. ) -> None:
  571. match.disable()
  572. if match.source:
  573. source_match = self.get_match(match.source)
  574. source_match.references -= 1
  575. def _set_match(
  576. self,
  577. match: Match,
  578. updated: int,
  579. source: Path,
  580. source_match: Match = None,
  581. dupplicate: bool = False,
  582. ) -> None:
  583. # source_match is in parameters because most of the time
  584. # its parent function needs it too,
  585. # so it can pass it to save a traversal
  586. source_match = source_match or self.get_match(source)
  587. new_level = source_match.level + 1
  588. if (
  589. updated > match.updated
  590. or new_level < match.level
  591. or source_match.first_party > match.first_party
  592. ):
  593. # NOTE FP and level of matches referencing this one
  594. # won't be updated until run or prune
  595. if match.source:
  596. old_source = self.get_match(match.source)
  597. old_source.references -= 1
  598. match.updated = updated
  599. match.level = new_level
  600. match.first_party = source_match.first_party
  601. match.source = source
  602. source_match.references += 1
  603. match.dupplicate = dupplicate
  604. def _set_domain(
  605. self, hostname: bool, domain_str: str, updated: int, source: Path
  606. ) -> None:
  607. self.enter_step("set_domain_val")
  608. if not Database.validate_domain(domain_str):
  609. raise ValueError(f"Invalid domain: {domain_str}")
  610. self.enter_step("set_domain_pack")
  611. domain = self.pack_domain(domain_str)
  612. self.enter_step("set_domain_fp")
  613. source_match = self.get_match(source)
  614. is_first_party = source_match.first_party
  615. self.enter_step("set_domain_brws")
  616. dic = self.domtree
  617. dupplicate = False
  618. for part in domain.parts:
  619. if part not in dic.children:
  620. dic.children[part] = DomainTreeNode()
  621. dic = dic.children[part]
  622. if dic.match_zone.active(is_first_party):
  623. dupplicate = True
  624. if hostname:
  625. match = dic.match_hostname
  626. else:
  627. match = dic.match_zone
  628. self._set_match(
  629. match,
  630. updated,
  631. source,
  632. source_match=source_match,
  633. dupplicate=dupplicate,
  634. )
  635. def set_hostname(self, *args: typing.Any, **kwargs: typing.Any) -> None:
  636. self._set_domain(True, *args, **kwargs)
  637. def set_zone(self, *args: typing.Any, **kwargs: typing.Any) -> None:
  638. self._set_domain(False, *args, **kwargs)
  639. def set_asn(self, asn_str: str, updated: int, source: Path) -> None:
  640. self.enter_step("set_asn")
  641. path = self.pack_asn(asn_str)
  642. if path.asn in self.asns:
  643. match = self.asns[path.asn]
  644. else:
  645. match = AsnNode()
  646. self.asns[path.asn] = match
  647. self._set_match(
  648. match,
  649. updated,
  650. source,
  651. )
  652. def _set_ip4(self, ip4: Ip4Path, updated: int, source: Path) -> None:
  653. self.enter_step("set_ip4_fp")
  654. source_match = self.get_match(source)
  655. is_first_party = source_match.first_party
  656. self.enter_step("set_ip4_brws")
  657. dic = self.ip4tree
  658. dupplicate = False
  659. for i in range(31, 31 - ip4.prefixlen, -1):
  660. bit = (ip4.value >> i) & 0b1
  661. next_dic = dic.one if bit else dic.zero
  662. if next_dic is None:
  663. next_dic = IpTreeNode()
  664. if bit:
  665. dic.one = next_dic
  666. else:
  667. dic.zero = next_dic
  668. dic = next_dic
  669. if dic.active(is_first_party):
  670. dupplicate = True
  671. self._set_match(
  672. dic,
  673. updated,
  674. source,
  675. source_match=source_match,
  676. dupplicate=dupplicate,
  677. )
  678. self._set_ip4cache(ip4, dic)
  679. def set_ip4address(
  680. self, ip4address_str: str, *args: typing.Any, **kwargs: typing.Any
  681. ) -> None:
  682. self.enter_step("set_ip4add_val")
  683. if not Database.validate_ip4address(ip4address_str):
  684. raise ValueError(f"Invalid ip4address: {ip4address_str}")
  685. self.enter_step("set_ip4add_pack")
  686. ip4 = self.pack_ip4address(ip4address_str)
  687. self._set_ip4(ip4, *args, **kwargs)
  688. def set_ip4network(
  689. self, ip4network_str: str, *args: typing.Any, **kwargs: typing.Any
  690. ) -> None:
  691. self.enter_step("set_ip4net_val")
  692. if not Database.validate_ip4network(ip4network_str):
  693. raise ValueError(f"Invalid ip4network: {ip4network_str}")
  694. self.enter_step("set_ip4net_pack")
  695. ip4 = self.pack_ip4network(ip4network_str)
  696. self._set_ip4(ip4, *args, **kwargs)