Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

68 lines
1.9 KiB

4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
4 months ago
  1. #!/usr/bin/env python3
  2. import database
  3. import argparse
  4. import requests
  5. import typing
  6. import ipaddress
  7. import logging
  8. import time
  9. IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
  10. def get_ranges(asn: str) -> typing.Iterable[str]:
  11. req = requests.get(
  12. "https://stat.ripe.net/data/as-routing-consistency/data.json",
  13. params={"resource": asn},
  14. )
  15. data = req.json()
  16. for pref in data["data"]["prefixes"]:
  17. yield pref["prefix"]
  18. def get_name(asn: str) -> str:
  19. req = requests.get(
  20. "https://stat.ripe.net/data/as-overview/data.json", params={"resource": asn}
  21. )
  22. data = req.json()
  23. return data["data"]["holder"]
  24. if __name__ == "__main__":
  25. log = logging.getLogger("feed_asn")
  26. # Parsing arguments
  27. parser = argparse.ArgumentParser(
  28. description="Add the IP ranges associated to the AS in the database"
  29. )
  30. args = parser.parse_args()
  31. DB = database.Database()
  32. def add_ranges(
  33. path: database.Path,
  34. match: database.Match,
  35. ) -> None:
  36. assert isinstance(path, database.AsnPath)
  37. assert isinstance(match, database.AsnNode)
  38. asn_str = database.Database.unpack_asn(path)
  39. DB.enter_step("asn_get_name")
  40. name = get_name(asn_str)
  41. match.name = name
  42. DB.enter_step("asn_get_ranges")
  43. for prefix in get_ranges(asn_str):
  44. parsed_prefix: IPNetwork = ipaddress.ip_network(prefix)
  45. if parsed_prefix.version == 4:
  46. DB.set_ip4network(prefix, source=path, updated=int(time.time()))
  47. log.info("Added %s from %s (%s)", prefix, path, name)
  48. elif parsed_prefix.version == 6:
  49. log.warning("Unimplemented prefix version: %s", prefix)
  50. else:
  51. log.error("Unknown prefix version: %s", prefix)
  52. for _ in DB.exec_each_asn(add_ranges):
  53. pass
  54. DB.save()