From 00a002091448b86fb9f24260a186172179842080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Thu, 14 Nov 2019 06:29:24 +0100 Subject: [PATCH] Added some delay for websites subdomains collecting Some websites load their trackers after the page is done loading. --- collect_subdomains.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collect_subdomains.py b/collect_subdomains.py index d55e883..4877d19 100755 --- a/collect_subdomains.py +++ b/collect_subdomains.py @@ -9,6 +9,7 @@ accessed by the websites. import sys import typing import urllib.parse +import time import progressbar import selenium.webdriver.firefox.options @@ -34,6 +35,7 @@ def collect_subdomains(url: str) -> typing.Iterable[str]: executable_path='geckodriver', options=options) driver.get(url) + time.sleep(10) for request in driver.requests: if request.response: yield subdomain_from_url(request.path)