|
|
@ -1,5 +1,9 @@ |
|
|
|
#!/usr/bin/env bash |
|
|
|
|
|
|
|
function log() { |
|
|
|
echo -e "\033[33m$@\033[0m" |
|
|
|
} |
|
|
|
|
|
|
|
if [ ! -f temp/all_resolved.csv ] |
|
|
|
then |
|
|
|
echo "Run ./resolve_subdomains.sh first!" |
|
|
@ -7,7 +11,7 @@ then |
|
|
|
fi |
|
|
|
|
|
|
|
# Gather all the rules for filtering |
|
|
|
echo "Compiling rules..." > /dev/stderr |
|
|
|
log "Compiling rules…" |
|
|
|
cat rules_adblock/*.txt | grep -v '^!' | grep -v '^\[Adblock' | sort -u > temp/all_rules_adblock.txt |
|
|
|
./adblock_to_domain_list.py --input temp/all_rules_adblock.txt --output rules/from_adblock.cache.list |
|
|
|
cat rules_hosts/*.txt | grep -v '^#' | grep -v '^$' | cut -d ' ' -f2 > rules/from_hosts.cache.list |
|
|
@ -16,19 +20,19 @@ cat rules/first-party.list | grep -v '^#' | grep -v '^$' | sort -u > temp/all_ru |
|
|
|
cat rules_ip/*.txt | grep -v '^#' | grep -v '^$' | sort -u > temp/all_ip_rules_multi.txt |
|
|
|
cat rules_ip/first-party.txt | grep -v '^#' | grep -v '^$' | sort -u > temp/all_ip_rules_first.txt |
|
|
|
|
|
|
|
echo "Filtering first-party tracking domains..." > /dev/stderr |
|
|
|
log "Filtering first-party tracking domains…" |
|
|
|
./filter_subdomains.py --rules temp/all_rules_first.list --rules-ip temp/all_ip_rules_first.txt --input temp/all_resolved_sorted.csv --output temp/firstparty-trackers.list |
|
|
|
sort -u temp/firstparty-trackers.list > dist/firstparty-trackers.txt |
|
|
|
|
|
|
|
echo "Filtering first-party curated tracking domains..." > /dev/stderr |
|
|
|
log "Filtering first-party curated tracking domains…" |
|
|
|
./filter_subdomains.py --rules temp/all_rules_first.list --rules-ip temp/all_ip_rules_first.txt --input temp/all_resolved_sorted.csv --no-explicit --output temp/firstparty-only-trackers.list |
|
|
|
sort -u temp/firstparty-only-trackers.list > dist/firstparty-only-trackers.txt |
|
|
|
|
|
|
|
echo "Filtering multi-party tracking domains..." > /dev/stderr |
|
|
|
log "Filtering multi-party tracking domains…" |
|
|
|
./filter_subdomains.py --rules temp/all_rules_multi.list --rules-ip temp/all_ip_rules_multi.txt --input temp/all_resolved_sorted.csv --output temp/multiparty-trackers.list |
|
|
|
sort -u temp/multiparty-trackers.list > dist/multiparty-trackers.txt |
|
|
|
|
|
|
|
echo "Filtering multi-party curated tracking domains..." > /dev/stderr |
|
|
|
log "Filtering multi-party curated tracking domains…" |
|
|
|
./filter_subdomains.py --rules temp/all_rules_multi.list --rules-ip temp/all_ip_rules_multi.txt --input temp/all_resolved_sorted.csv --no-explicit --output temp/multiparty-only-trackers.list |
|
|
|
sort -u temp/multiparty-only-trackers.list > dist/multiparty-only-trackers.txt |
|
|
|
|
|
|
|