pattern based host whitelisting / rule priorities (1f792337) · Commits · e / os / PrivacyModule TrackerFilter

app/src/main/assets/additionalHosts.txt

+18 −9

Original line number	Original line	Diff line number	Diff line
	# Add your own additional hosts here!		# Add your own additional hosts here!
	# Supports Blacklist, whitelist and map to custom IP		# Supports blacklist, whitelist and map to custom IP

	# Format for Blacklist: 1 host name per line (sample below).		# WARNING! You must not add huge lists here!
			# This is only for overruling the configured filter.

			# Format for blacklist: 1 host name per line (sample below).
	# Wildcard Character "*" can be used for host blocking.		# Wildcard Character "*" can be used for host blocking.
	# Host name and all sub domains will be blocked.		# Host name and all sub domains will be blocked,
			# unless there is a more specific rule.

	sample4711.justasample.com		sample4711.justasample.com
	sample4712.justasample.com		sample4712.justasample.com

	# In order to white list a specific host, add a		# In order to white list a specific host, use "!" prefix.
	# "!" prefix in front of the host name.		# Wildcard Character "*" can be used for whitelisting.
	# Whitelisting does NOT support wildcards "*"		# Host name and all sub domains will be whitelisted,
			# unless there is a more specific rule.

	!whitelistsample.justasample.com		!whitelistsample.justasample.com

	# In order to forward to custom ip add a ">" in front		# PRIORITIES in case of conflicting entries
	# of the host name followed by whitespace and mapped IP		# 1. Explicit entries without wildcards
			# 2. Wildcard entries prioritized along the sequence in this file
			# 3. Downloaded blocklist

	>router.home 192.168.2.1		# In order to forward to custom ip add a ">" in front of the host,
			# followed by whitespace and mapped IP.

			>router.home 192.168.2.1

app/src/main/assets/dnsfilter.conf

+0 −4

Original line number	Original line	Diff line number	Diff line
	@@ -176,10 +176,6 @@ filterAutoUpdateURL_switchs = true; true; false; false; false; false; false; fal
	# DO NOT CHANGE! - will be set internally!		# DO NOT CHANGE! - will be set internally!
	previousAutoUpdateURL = <none>		previousAutoUpdateURL = <none>

	# additionalHosts_lastImportTS - the time stamp of additionalHosts.txt when it was last imported
	# DO NOT CHANGE! - will be set internally!
	additionalHosts_lastImportTS = 0

	# reloadIntervalDays - specifies the number of days, after the filter gets updated when configured.		# reloadIntervalDays - specifies the number of days, after the filter gets updated when configured.
	reloadIntervalDays =7		reloadIntervalDays =7

app/src/main/java/dnsfilter/BlockedHosts.java

+136 −138

Original line number	Original line	Diff line number	Diff line
	@@ -67,62 +67,153 @@ public class BlockedHosts implements Set {
	private static Object NOT_NULL = new Object();		private static Object NOT_NULL = new Object();
	private LRUCache okCache;		private LRUCache okCache;
	private LRUCache filterListCache;		private LRUCache filterListCache;
	private Hashtable hostsFilterOverRule;

	private int sharedLocks = 0;		private int sharedLocks = 0;
	private boolean exclusiveLock = false;		private boolean exclusiveLock = false;

			private Hashtable <String, Boolean> hostsFilterOverRule = new Hashtable<String, Boolean>();

	private HugePackedSet blockedHostsHashes;		private HugePackedSet blockedHostsHashes;
	private Vector<String[]> blockedPatterns;
	private Vector<String[]> allowedPatterns;

	public BlockedHosts(int maxCountEstimate, int okCacheSize, int filterListCacheSize, Hashtable hostsFilterOverRule) {		private class OverrulePattern {
			private String[] pattern;
			private boolean filter;
			private int hashcode;
			private OverrulePattern (String patternString, boolean filter){
			this.pattern = patternString.split("\\*", -1);
			this.filter = filter;
			hashcode = patternString.hashCode();
			if (!filter)
			hashcode = ~hashcode;
			}

			@Override
			public int hashCode() {
			return hashcode;
			}

			@Override
			public boolean equals(Object obj) {
			if (obj != null && hashcode == obj.hashCode() && obj instanceof OverrulePattern)
			return patternEqual((OverrulePattern) obj);
			else
			return false;
			}

			private boolean patternEqual(OverrulePattern overrulePattern) {
			if (pattern.length == overrulePattern.pattern.length) {
			for (int i = 0; i < pattern.length; i++) {
			if (!pattern[i].equals(overrulePattern.pattern[i]))
			return false;
			}
			return true;
			} else
			return false;
			}

			private boolean match(String host) {

			// Iterate over the parts.
			for (int i = 0; i < pattern.length; i++) {
			String part = pattern[i];

			int idx = -1;
			if (i < pattern.length-1)
			idx = host.indexOf(part);
			else
			idx = host.lastIndexOf(part);

			if (i == 0 && !part.equals("") && idx != 0) {
			// i == 0 ==> we are on the first fixed part
			// first fixed part is not empty ==> Matching String must start with first fixed part
			// if not, no match!
			return false;
			}
			if (i == pattern.length-1 && !part.equals("") && idx + part.length() != host.length()) {
			// i == last part
			// last part is not empty ==> Matching String must end with last part
			// if not, no match
			return false;
			}
			// part not detected in the text.
			if (idx == -1) {
			return false;
			}
			// Move ahead, towards the right of the text.
			host = host.substring(idx + part.length());
			}
			return true;
			}

			}
			private Vector<OverrulePattern> overrulePatterns = new Vector<OverrulePattern>();

			public BlockedHosts(int maxCountEstimate, int okCacheSize, int filterListCacheSize) {
	okCache = new LRUCache(okCacheSize);		okCache = new LRUCache(okCacheSize);
	filterListCache = new LRUCache(filterListCacheSize);		filterListCache = new LRUCache(filterListCacheSize);

	if (ExecutionEnvironment.getEnvironment().debug())		if (ExecutionEnvironment.getEnvironment().debug())
	Logger.getLogger().logLine("CACHE SIZE:"+okCacheSize+", "+filterListCacheSize);		Logger.getLogger().logLine("CACHE SIZE:"+okCacheSize+", "+filterListCacheSize);

	this.hostsFilterOverRule = hostsFilterOverRule;

	int slots = maxCountEstimate / 6000;		int slots = maxCountEstimate / 6000;
	if ((slots % 2) == 0)		if ((slots % 2) == 0)
	slots++;		slots++;

	blockedHostsHashes = new HugePackedSet(slots, PACK_MGR);		blockedHostsHashes = new HugePackedSet(slots, PACK_MGR);
	buildAllowedPatternStruct();
	}		}

	private BlockedHosts(HugePackedSet blockedHostsHashes, Vector<String[]> blockedPatterns, int okCacheSize, int filterListCacheSize, Hashtable hostsFilterOverRule) {		private BlockedHosts(HugePackedSet blockedHostsHashes, int okCacheSize, int filterListCacheSize) {
	this.blockedHostsHashes = blockedHostsHashes;		this.blockedHostsHashes = blockedHostsHashes;
	this.blockedPatterns = blockedPatterns;
	okCache = new LRUCache(okCacheSize);		okCache = new LRUCache(okCacheSize);
	filterListCache = new LRUCache(filterListCacheSize);		filterListCache = new LRUCache(filterListCacheSize);

	if (ExecutionEnvironment.getEnvironment().debug())		if (ExecutionEnvironment.getEnvironment().debug())
	Logger.getLogger().logLine("CACHE SIZE:"+okCacheSize+", "+filterListCacheSize);		Logger.getLogger().logLine("CACHE SIZE:"+okCacheSize+", "+filterListCacheSize);
			}


	this.hostsFilterOverRule = hostsFilterOverRule;		public void addOverrule(String host, boolean filter) {
	buildAllowedPatternStruct();		host = host.toLowerCase();

			if (host.indexOf("*") != -1) {
			overrulePatterns.add(new OverrulePattern(host, filter));
			clearCache(filter);
			}
			else {
			hostsFilterOverRule.put(host, new Boolean(filter));
			clearCache(host, filter);
			}
	}		}

	private void buildAllowedPatternStruct(){		public void removeOverrule (String host, boolean filter) {
	if (hostsFilterOverRule == null)		host = host.toLowerCase();
	return;
	String[] entries = (String[]) hostsFilterOverRule.keySet().toArray(new String[hostsFilterOverRule.size()]);
	for (int i = 0; i < entries.length; i++){
	String entry = entries[i];
	boolean filter = ((Boolean) hostsFilterOverRule.get(entry)).booleanValue();

	if (!filter && entry.indexOf("*") != -1)		if (host.indexOf("*") != -1) {
	getInitializedPatternStruct(ALLOWED_PATTERN).addElement( ((String)entry).trim().toLowerCase().split("\\*", -1));		overrulePatterns.remove(new OverrulePattern(host, filter));
			clearCache(!filter);
			}
			else {
			Boolean val = hostsFilterOverRule.get(host);
			if (val != null && val.booleanValue() == filter) {
			hostsFilterOverRule.remove(host);
			clearCache(host, !filter);
			}
			}
	}		}

			private void clearCache(String host, boolean filter) {
			long hostHash = Utils.getLongStringHash((String) host);
			if (filter)
			okCache.remove(hostHash);
			else
			filterListCache.remove(hostHash);
	}		}

	public void setHostsFilterOverRule(Hashtable hostsFilterOverRule){		private void clearCache(boolean filter) {
	if (hostsFilterOverRule == null)		if (filter)
	throw new IllegalArgumentException("Argument null not allowed!");		okCache.clear();
	this.hostsFilterOverRule = hostsFilterOverRule;		else
			filterListCache.clear();
	}		}


	@@ -170,39 +261,15 @@ public class BlockedHosts implements Set {
	return HugePackedSet.checkIndexVersion(path);		return HugePackedSet.checkIndexVersion(path);
	}		}

	public static BlockedHosts loadPersistedIndex(String path, boolean inMemory, int okCacheSize, int filterListCacheSize, Hashtable hostsFilterOverRule) throws IOException {		public static BlockedHosts loadPersistedIndex(String path, boolean inMemory, int okCacheSize, int filterListCacheSize) throws IOException {
	Vector<String[]> blockedPatterns = null;		return new BlockedHosts(HugePackedSet.load(path, inMemory, PACK_MGR), okCacheSize, filterListCacheSize);
	File patternFile = new File(path+"/blockedpatterns");
	if (patternFile.exists()){
	BufferedReader fin = new BufferedReader(new InputStreamReader(new FileInputStream(patternFile)));
	blockedPatterns = new Vector<String[]>();
	String entry;
	while ( (entry = fin.readLine()) != null) {
	blockedPatterns.addElement( ((String)entry).trim().split("\\*", -1));
	}
	fin.close();
	}
	return new BlockedHosts(HugePackedSet.load(path, inMemory, PACK_MGR), blockedPatterns, okCacheSize, filterListCacheSize, hostsFilterOverRule);
	}		}


	public void persist(String path) throws IOException {		public void persist(String path) throws IOException {
	try {		try {
	lock(1);		lock(1);
	blockedHostsHashes.persist(path);		blockedHostsHashes.persist(path);
	if (blockedPatterns != null) {
	OutputStream patterns = new BufferedOutputStream(new FileOutputStream(path + "/blockedpatterns"));
	Iterator it = blockedPatterns.iterator();
	while (it.hasNext()) {
	String[] fixedParts = (String[]) it.next();
	String patternStr = fixedParts[0];
	for (int i = 1; i < fixedParts.length; i++) {
	patternStr = patternStr + "*" + fixedParts[i];
	}
	patterns.write((patternStr + "\n").getBytes());
	}
	patterns.flush();
	patterns.close();
	}
	} finally {		} finally {
	unLock(1);		unLock(1);
	}		}
	@@ -232,30 +299,6 @@ public class BlockedHosts implements Set {
	}		}



	static int BLOCKED_PATTERN = 0;
	static int ALLOWED_PATTERN = 1;
	private Vector<String[]> getInitializedPatternStruct(int type) {
	if (type == BLOCKED_PATTERN) {
	if (blockedPatterns == null)
	blockedPatterns = new Vector<String[]>();
	return blockedPatterns;
	}
	else if (type == ALLOWED_PATTERN) {
	if (allowedPatterns == null)
	allowedPatterns = new Vector<String[]>();
	return allowedPatterns;
	}
	else return null;
	}


	public void clearCache(String host) {
	long hostHash = Utils.getLongStringHash((String) host.toLowerCase());
	okCache.remove(hostHash);
	filterListCache.remove(hostHash);
	}

	public boolean update(Object host) throws IOException {		public boolean update(Object host) throws IOException {
	try {		try {
	lock(1);		lock(1);
	@@ -275,61 +318,21 @@ public class BlockedHosts implements Set {

	@Override		@Override
	public boolean add(Object host) {		public boolean add(Object host) {
	if (((String) host).indexOf("*") == -1)
	return blockedHostsHashes.add(Utils.getLongStringHash((String) ((String) host).toLowerCase()));		return blockedHostsHashes.add(Utils.getLongStringHash((String) ((String) host).toLowerCase()));
	else { //Pattern
	getInitializedPatternStruct(BLOCKED_PATTERN).addElement( ((String)host).trim().toLowerCase().split("\\*", -1));
	return true;
	}
	}		}

	private boolean containsPatternMatch(String host, int type) {

	if ( (type == BLOCKED_PATTERN && blockedPatterns == null) \|\| (type == ALLOWED_PATTERN && allowedPatterns == null))		private OverrulePattern getOverrulePattern (String host) {
	return false;

	Iterator it = getInitializedPatternStruct(type).iterator();		Iterator<OverrulePattern> it = overrulePatterns.iterator();
	while (it.hasNext()) {		while (it.hasNext()) {
	String[] fixedParts = (String[]) it.next();		OverrulePattern pattern = it.next();
	if (wildCardMatch(fixedParts, host))		if (pattern.match(host))
	return true;		return pattern;
	}		}
	return false;		return null;
	}		}

	private static boolean wildCardMatch(String[] fixedParts, String host) {

	// Iterate over the parts.
	for (int i = 0; i < fixedParts.length; i++) {
	String part = fixedParts[i];

	int idx = -1;
	if (i < fixedParts.length-1)
	idx = host.indexOf(part);
	else
	idx = host.lastIndexOf(part);

	if (i == 0 && !part.equals("") && idx != 0) {
	// i == 0 ==> we are on the first fixed part
	// first fixed part is not empty ==> Matching String must start with first fixed part
	// if not, no match!
	return false;
	}
	if (i == fixedParts.length-1 && !part.equals("") && idx + part.length() != host.length()) {
	// i == last part
	// last part is not empty ==> Matching String must end with last part
	// if not, no match
	return false;
	}
	// part not detected in the text.
	if (idx == -1) {
	return false;
	}
	// Move ahead, towards the right of the text.
	host = host.substring(idx + part.length());
	}
	return true;
	}

	@Override		@Override
	public boolean contains(Object object) {		public boolean contains(Object object) {
	@@ -367,20 +370,19 @@ public class BlockedHosts implements Set {

	while (idx != -1) {		while (idx != -1) {

	if (hostsFilterOverRule != null) {		Boolean filter = hostsFilterOverRule.get(hostName);
	Object val = hostsFilterOverRule.get(hostName);		if (filter != null)
	if (val != null)		return filter.booleanValue();
	return ((Boolean) val).booleanValue();
	}

	if (checkPattern && containsPatternMatch(hostName, ALLOWED_PATTERN))
	return false;

	if (checkPattern && containsPatternMatch(hostName, BLOCKED_PATTERN))		if (checkPattern) {
	return true;		OverrulePattern patternMatch = getOverrulePattern(hostName);
			if (patternMatch != null)
			return patternMatch.filter;
			}

	if (blockedHostsHashes.contains(hosthash))		if (blockedHostsHashes.contains(hosthash))
	return true;		return true;

	if (checkParent) {		if (checkParent) {
	idx = hostName.indexOf('.');		idx = hostName.indexOf('.');

	@@ -398,10 +400,7 @@ public class BlockedHosts implements Set {
	filterListCache.clear();		filterListCache.clear();
	okCache.clear();		okCache.clear();
	hostsFilterOverRule = null; // do not clear as provided from outside and reused		hostsFilterOverRule = null; // do not clear as provided from outside and reused
	if (blockedPatterns != null)		overrulePatterns = null; // do not clear as provided from outside and reused
	blockedPatterns.clear();
	if (allowedPatterns != null)
	allowedPatterns.clear();
	}		}

	protected void migrateTo(BlockedHosts hostFilter) {		protected void migrateTo(BlockedHosts hostFilter) {
	@@ -415,8 +414,7 @@ public class BlockedHosts implements Set {

	hostsFilterOverRule = hostFilter.hostsFilterOverRule;		hostsFilterOverRule = hostFilter.hostsFilterOverRule;

	blockedPatterns = hostFilter.blockedPatterns;		overrulePatterns = hostFilter.overrulePatterns;
	allowedPatterns = hostFilter.allowedPatterns;

	//blockedHostsHashes.migrateTo(hostFilter.blockedHostsHashes);		//blockedHostsHashes.migrateTo(hostFilter.blockedHostsHashes);
	blockedHostsHashes = hostFilter.blockedHostsHashes;		blockedHostsHashes = hostFilter.blockedHostsHashes;

app/src/main/java/dnsfilter/DNSFilterManager.java

+109 −159

File changed.

Preview size limit exceeded, changes collapsed.