Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

70 lines
1.9 KiB

  1. #!/usr/bin/env python3
  2. import database
  3. import argparse
  4. import requests
  5. import typing
  6. import ipaddress
  7. import logging
  8. import time
  9. IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
  10. def get_ranges(asn: str) -> typing.Iterable[str]:
  11. req = requests.get(
  12. 'https://stat.ripe.net/data/as-routing-consistency/data.json',
  13. params={'resource': asn}
  14. )
  15. data = req.json()
  16. for pref in data['data']['prefixes']:
  17. yield pref['prefix']
  18. def get_name(asn: str) -> str:
  19. req = requests.get(
  20. 'https://stat.ripe.net/data/as-overview/data.json',
  21. params={'resource': asn}
  22. )
  23. data = req.json()
  24. return data['data']['holder']
  25. if __name__ == '__main__':
  26. log = logging.getLogger('feed_asn')
  27. # Parsing arguments
  28. parser = argparse.ArgumentParser(
  29. description="TODO")
  30. args = parser.parse_args()
  31. DB = database.Database()
  32. def add_ranges(path: database.Path,
  33. match: database.Match,
  34. ) -> None:
  35. assert isinstance(path, database.AsnPath)
  36. assert isinstance(match, database.AsnNode)
  37. asn_str = database.Database.unpack_asn(path)
  38. DB.enter_step('asn_get_name')
  39. match.name = get_name(asn_str)
  40. DB.enter_step('asn_get_ranges')
  41. for prefix in get_ranges(asn_str):
  42. parsed_prefix: IPNetwork = ipaddress.ip_network(prefix)
  43. if parsed_prefix.version == 4:
  44. DB.set_ip4network(
  45. prefix,
  46. source=path,
  47. updated=int(time.time())
  48. )
  49. log.info('Added %s from %s (%s)', prefix, asn_str, path)
  50. elif parsed_prefix.version == 6:
  51. log.warning('Unimplemented prefix version: %s', prefix)
  52. else:
  53. log.error('Unknown prefix version: %s', prefix)
  54. for _ in DB.exec_each_asn(add_ranges):
  55. pass
  56. DB.save()