Convert script to nodejs (8328dbbd) · Commits · e / os / tracker-list

adaway_hosts.txt

deleted100644 → 0

+0 −11351

File deleted.

Preview size limit exceeded, changes collapsed.

build_list.html

deleted100644 → 0

+0 −157

Original line number	Diff line number	Diff line
		<!DOCTYPE html>
		<html lang="fr">
		<head>
		<meta charset="utf-8">
		</head>
		<body>
		<h1> Hello </h1>

		<script>
		let adawayText = null
		let exodusJson = null

		let hostnames = []

		let trackers = []

		document.body.append("script OK")

		// https://adaway.org/hosts.txt
		fetch("./adaway_hosts.txt")
		.then(response => {
		return response.text()
		}).then(text => {
		adawayText = text
		hostnames = parseHostToHostnames(text)
		document.body.append("adaway OK")
		}).then(toto => {
		// https://reports.exodus-privacy.eu.org/api/trackers
		return fetch("exodus_trackers.json")
		})
		.then(response => { return response.json()})
		.then(json => {
		exoduxJson = json
		document.body.append("exodus OK")
		})
		.then(toto => {
		trackers = exodusToETrackers(exoduxJson)

		hostnames = applyOutbrainRule(trackers, hostnames)
		hostnames = enrichExodusWithAdaway(trackers, hostnames)
		document.body.append("enrich exodus OK")
		hostnames = createdLevel2Trackers(trackers, hostnames)
		document.body.append("DONE")
		})
		.then(toto => {
		printTrackers()
		})


		function parseHostToHostnames(hostText) {
		return hostText
		.split("\n")
		.filter(line => { return !line.startsWith("#") && line != ""})
		.map(line => line.split(" ")[1])
		.filter(it => it != "")
		}

		function exodusToETrackers(exodusTrackers) {
		return Object.values(exodusTrackers.trackers)
		.filter(tracker => tracker.network_signature != "")
		.map(tracker => {
		const id = "exodus_" + tracker.id
		const hostnames = tracker.network_signature
		.replaceAll("\\", "")
		.split("\|")
		return {
		id: id,
		hostnames: hostnames,
		name: tracker.name,
		exodusId: tracker.id
		}
		})
		}


		function applyOutbrainRule(trackersList, adawayList) {
		const tracker = trackersList.find(it => it.name = "OutBrain")
		if (tracker != null) {
		return extractSubhostnamesToTracker('outbrainimg.com', tracker, adawayList)
		}
		return adawayList
		}

		function extractSubhostnamesToTracker(hostname, tracker, adawayList) {
		let hostnamesSet = new Set(tracker.hostnames)
		adawayList.filter(it => it.endsWith(hostname))
		.forEach(subhost => hostnamesSet.add(subhost))
		tracker.hostnames = Array.from(hostnamesSet)
		return adawayList.filter(it => !it.endsWith(hostname))
		}



		function enrichExodusWithAdaway(trackersList, adawayList) {
		trackersList.forEach(tracker => {
		tracker.hostnames.forEach(hostname => {
		adawayList = extractSubhostnamesToTracker(hostname, tracker, adawayList)
		})
		})
		return adawayList
		}

		function createdLevel2Trackers(trackersList, adawayList) {

		while(adawayList.length > 0) {
		let hostname = adawayList[0]
		//return adawayList.reduce((acc, hostname) => {

		let lv2 = toLevel2Domain(hostname)
		let tracker = {
		id: "adaway_" + lv2,
		name: lv2,
		hostnames: []
		}
		trackersList.push(tracker)
		adawayList = extractSubhostnamesToTracker(lv2, tracker, adawayList)
		}
		return adawayList
		}

		function toLevel2Domain(hostname) {
		try {
		let parts = hostname.split("\.")
		return parts.slice(parts.length - 2, parts.length).join(".")
		} catch {
		console.log(hostname)
		}
		}

		function printTrackers() {
		document.body.innerHTML = JSON.stringify({
		trackers: trackers,
		createdAt: new Date().toISOString()
		})
		}

		// Convert exodus to new format

		// Apply custom rules (like outbrain / outbraining)

		// Enrich exodus with adaway: add to an exodus tracker, each adaway hostname,
		// which is a subdomain of one of the exodus hostnames.

		// group remaining hostname by 2 level hostnames, and create trackers.

		// Result struct
		// {
		// trackers: [
		//
		// val id: String,
		// val hostnames: Set<String>,
		// val label: String,
		// val exodusId: String?

		</script>
		</body>
		</html>
		No newline at end of file

build_list.js

+154 −1

Original line number	Diff line number	Diff line
		console.log("Hello nodejs")
		No newline at end of file
		const outputFile = "list/e_trackers.json"

		const adawayUri = "https://adaway.org/hosts.txt"
		const exodusUri = "https://reports.exodus-privacy.eu.org/api/trackers"


		const https = require('https')
		const fs = require('fs')
		//const process = require('process')

		function fetch(url) {
		return new Promise((resolve, reject) => {
		https.get(url, (response) => {
		let data = ""
		response.on('data', (chunk) => { data += chunk })
		response.on('end', () => resolve(data))
		}).on('error', reject)
		})
		}


		let adawayText = null
		let exodusJson = null

		let hostnames = []

		let trackers = []

		console.log("Starting list update")

		fetch(adawayUri)
		//.then(response => {
		// return response.text()
		//})
		.then(text => {
		adawayText = text
		hostnames = parseHostToHostnames(text)
		console.log("adaway OK")
		}).then(toto => {
		return fetch(exodusUri)
		})
		//.then(response => { return response.json()})
		.then(response => { return JSON.parse(response) })
		.then(json => {
		exoduxJson = json
		console.log("exodus OK")
		})
		.then(toto => {
		trackers = exodusToETrackers(exoduxJson)

		hostnames = applyOutbrainRule(trackers, hostnames)
		hostnames = enrichExodusWithAdaway(trackers, hostnames)
		console.log("enrich exodus OK")
		hostnames = createdLevel2Trackers(trackers, hostnames)

		if (trackers.length < 400) {
		throw Exception(`Not enough trackers (${trackers.length}), abort`)
		}

		console.log("DONE")
		})
		.then(toto => {
		printTrackers()
		})
		.catch(err => {
		console.log(err)
		process.exit(1)
		})


		function parseHostToHostnames(hostText) {
		return hostText
		.split("\n")
		.filter(line => { return !line.startsWith("#") && line != ""})
		.map(line => line.split(" ")[1])
		.filter(it => it != "")
		}

		function exodusToETrackers(exodusTrackers) {
		return Object.values(exodusTrackers.trackers)
		.filter(tracker => tracker.network_signature != "")
		.map(tracker => {
		const id = "exodus_" + tracker.id
		const hostnames = tracker.network_signature
		.replaceAll("\\", "")
		.split("\|")
		return {
		id: id,
		hostnames: hostnames,
		name: tracker.name,
		exodusId: tracker.id
		}
		})
		}


		function applyOutbrainRule(trackersList, adawayList) {
		const tracker = trackersList.find(it => it.name = "OutBrain")
		if (tracker != null) {
		return extractSubhostnamesToTracker('outbrainimg.com', tracker, adawayList)
		}
		return adawayList
		}

		function extractSubhostnamesToTracker(hostname, tracker, adawayList) {
		let hostnamesSet = new Set(tracker.hostnames)
		adawayList.filter(it => it.endsWith(hostname))
		.forEach(subhost => hostnamesSet.add(subhost))
		tracker.hostnames = Array.from(hostnamesSet)
		return adawayList.filter(it => !it.endsWith(hostname))
		}

		function enrichExodusWithAdaway(trackersList, adawayList) {
		trackersList.forEach(tracker => {
		tracker.hostnames.forEach(hostname => {
		adawayList = extractSubhostnamesToTracker(hostname, tracker, adawayList)
		})
		})
		return adawayList
		}

		function createdLevel2Trackers(trackersList, adawayList) {

		while(adawayList.length > 0) {
		let hostname = adawayList[0]
		//return adawayList.reduce((acc, hostname) => {

		let lv2 = toLevel2Domain(hostname)
		let tracker = {
		id: "adaway_" + lv2,
		name: lv2,
		hostnames: []
		}
		trackersList.push(tracker)
		adawayList = extractSubhostnamesToTracker(lv2, tracker, adawayList)
		}
		return adawayList
		}

		function toLevel2Domain(hostname) {
		try {
		let parts = hostname.split("\.")
		return parts.slice(parts.length - 2, parts.length).join(".")
		} catch {
		console.log(hostname)
		}
		}

		function printTrackers() {
		fs.writeFileSync(outputFile, JSON.stringify({
		trackers: trackers,
		createdAt: new Date().toISOString()
		}))
		}

exodus_trackers.json

deleted100644 → 0

+0 −1

File deleted.

Preview size limit exceeded, changes collapsed.

list/e_trackers.json

+1 −1

File changed.

Preview size limit exceeded, changes collapsed.