Generates a host list of first-party trackers for ad-blocking.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

72 lines
2.0KB

  1. #!/usr/bin/env python3
  2. import database
  3. import argparse
  4. import requests
  5. import typing
  6. import ipaddress
  7. import logging
  8. import time
  9. IPNetwork = typing.Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
  10. def get_ranges(asn: str) -> typing.Iterable[str]:
  11. req = requests.get(
  12. 'https://stat.ripe.net/data/as-routing-consistency/data.json',
  13. params={'resource': asn}
  14. )
  15. data = req.json()
  16. for pref in data['data']['prefixes']:
  17. yield pref['prefix']
  18. def get_name(asn: str) -> str:
  19. req = requests.get(
  20. 'https://stat.ripe.net/data/as-overview/data.json',
  21. params={'resource': asn}
  22. )
  23. data = req.json()
  24. return data['data']['holder']
  25. if __name__ == '__main__':
  26. log = logging.getLogger('feed_asn')
  27. # Parsing arguments
  28. parser = argparse.ArgumentParser(
  29. description="Add the IP ranges associated to the AS in the database")
  30. args = parser.parse_args()
  31. DB = database.Database()
  32. def add_ranges(path: database.Path,
  33. match: database.Match,
  34. ) -> None:
  35. assert isinstance(path, database.AsnPath)
  36. assert isinstance(match, database.AsnNode)
  37. asn_str = database.Database.unpack_asn(path)
  38. DB.enter_step('asn_get_name')
  39. name = get_name(asn_str)
  40. match.name = name
  41. DB.enter_step('asn_get_ranges')
  42. for prefix in get_ranges(asn_str):
  43. parsed_prefix: IPNetwork = ipaddress.ip_network(prefix)
  44. if parsed_prefix.version == 4:
  45. DB.set_ip4network(
  46. prefix,
  47. source=path,
  48. updated=int(time.time())
  49. )
  50. log.info('Added %s from %s (%s)', prefix, path, name)
  51. elif parsed_prefix.version == 6:
  52. log.warning('Unimplemented prefix version: %s', prefix)
  53. else:
  54. log.error('Unknown prefix version: %s', prefix)
  55. for _ in DB.exec_each_asn(add_ranges):
  56. pass
  57. DB.save()