Update domain substitution (3bfb9a3c) · Commits · e / os / Browser

domain_substitution/_common.py

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -36,6 +36,7 @@ class ExtractorEnum: #pylint: disable=too-few-public-methods

	class SetLogLevel(argparse.Action): #pylint: disable=too-few-public-methods		class SetLogLevel(argparse.Action): #pylint: disable=too-few-public-methods
	"""Sets logging level based on command line arguments it receives"""		"""Sets logging level based on command line arguments it receives"""

	def __init__(self, option_strings, dest, nargs=None, **kwargs):		def __init__(self, option_strings, dest, nargs=None, **kwargs):
	super().__init__(option_strings, dest, nargs=nargs, **kwargs)		super().__init__(option_strings, dest, nargs=nargs, **kwargs)

domain_substitution/_extraction.py

+18 −17

Original line number	Original line	Diff line number	Diff line
	@@ -80,7 +80,7 @@ def _process_relative_to(unpack_root, relative_to):
	if not relative_root.is_dir():		if not relative_root.is_dir():
	get_logger().error('Could not find relative_to directory in extracted files: %s',		get_logger().error('Could not find relative_to directory in extracted files: %s',
	relative_to)		relative_to)
	raise Exception()		raise FileNotFoundError()
	for src_path in relative_root.iterdir():		for src_path in relative_root.iterdir():
	dest_path = unpack_root / src_path.name		dest_path = unpack_root / src_path.name
	src_path.rename(dest_path)		src_path.rename(dest_path)
	@@ -92,20 +92,20 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to):
	if not relative_to is None and (output_dir / relative_to).exists():		if not relative_to is None and (output_dir / relative_to).exists():
	get_logger().error('Temporary unpacking directory already exists: %s',		get_logger().error('Temporary unpacking directory already exists: %s',
	output_dir / relative_to)		output_dir / relative_to)
	raise Exception()		raise FileExistsError()
	cmd1 = (binary, 'x', str(archive_path), '-so')		cmd1 = (binary, 'x', str(archive_path), '-so')
	cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(output_dir)))		cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', f'-o{str(output_dir)}')
	get_logger().debug('7z command line: %s \| %s', ' '.join(cmd1), ' '.join(cmd2))		get_logger().debug('7z command line: %s \| %s', ' '.join(cmd1), ' '.join(cmd2))

	proc1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE)		proc1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE) #pylint: disable=consider-using-with
	proc2 = subprocess.Popen(cmd2, stdin=proc1.stdout, stdout=subprocess.PIPE)		proc2 = subprocess.Popen(cmd2, stdin=proc1.stdout, stdout=subprocess.PIPE) #pylint: disable=consider-using-with
	proc1.stdout.close()		proc1.stdout.close()
	(stdout_data, stderr_data) = proc2.communicate()		(stdout_data, stderr_data) = proc2.communicate()
	if proc2.returncode != 0:		if proc2.returncode != 0:
	get_logger().error('7z commands returned non-zero status: %s', proc2.returncode)		get_logger().error('7z commands returned non-zero status: %s', proc2.returncode)
	get_logger().debug('stdout: %s', stdout_data)		get_logger().debug('stdout: %s', stdout_data)
	get_logger().debug('stderr: %s', stderr_data)		get_logger().debug('stderr: %s', stderr_data)
	raise Exception()		raise ChildProcessError()

	_process_relative_to(output_dir, relative_to)		_process_relative_to(output_dir, relative_to)

	@@ -118,7 +118,7 @@ def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to):
	result = subprocess.run(cmd, check=False)		result = subprocess.run(cmd, check=False)
	if result.returncode != 0:		if result.returncode != 0:
	get_logger().error('tar command returned %s', result.returncode)		get_logger().error('tar command returned %s', result.returncode)
	raise Exception()		raise ChildProcessError()

	# for gnu tar, the --transform option could be used. but to keep compatibility with		# for gnu tar, the --transform option could be used. but to keep compatibility with
	# bsdtar on macos, we just do this ourselves		# bsdtar on macos, we just do this ourselves
	@@ -133,7 +133,7 @@ def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to):
	result = subprocess.run(cmd, check=False)		result = subprocess.run(cmd, check=False)
	if result.returncode != 0:		if result.returncode != 0:
	get_logger().error('WinRAR command returned %s', result.returncode)		get_logger().error('WinRAR command returned %s', result.returncode)
	raise Exception()		raise ChildProcessError()

	_process_relative_to(output_dir, relative_to)		_process_relative_to(output_dir, relative_to)

	@@ -143,10 +143,12 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to):

	class NoAppendList(list):		class NoAppendList(list):
	"""Hack to workaround memory issues with large tar files"""		"""Hack to workaround memory issues with large tar files"""

	def append(self, obj):		def append(self, obj):
	pass		pass

	# Simple hack to check if symlinks are supported		# Simple hack to check if symlinks are supported
			symlink_supported = False
	try:		try:
	os.symlink('', '')		os.symlink('', '')
	except FileNotFoundError:		except FileNotFoundError:
	@@ -155,13 +157,12 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to):
	except OSError:		except OSError:
	# Symlinks probably not supported		# Symlinks probably not supported
	get_logger().info('System does not support symlinks. Ignoring them.')		get_logger().info('System does not support symlinks. Ignoring them.')
	symlink_supported = False
	except BaseException:		except BaseException:
	# Unexpected exception		# Unexpected exception
	get_logger().exception('Unexpected exception during symlink support check.')		get_logger().exception('Unexpected exception during symlink support check.')
	raise		raise

	with tarfile.open(str(archive_path), 'r\|%s' % archive_path.suffix[1:]) as tar_file_obj:		with tarfile.open(str(archive_path), f'r\|{archive_path.suffix[1:]}') as tar_file_obj:
	tar_file_obj.members = NoAppendList()		tar_file_obj.members = NoAppendList()
	for tarinfo in tar_file_obj:		for tarinfo in tar_file_obj:
	try:		try:
	@@ -258,21 +259,21 @@ def extract_with_7z(archive_path, output_dir, relative_to, extractors=None):
	if sevenzip_cmd == USE_REGISTRY:		if sevenzip_cmd == USE_REGISTRY:
	if not get_running_platform() == PlatformEnum.WINDOWS:		if not get_running_platform() == PlatformEnum.WINDOWS:
	get_logger().error('"%s" for 7-zip is only available on Windows', sevenzip_cmd)		get_logger().error('"%s" for 7-zip is only available on Windows', sevenzip_cmd)
	raise Exception()		raise EnvironmentError()
	sevenzip_cmd = str(_find_7z_by_registry())		sevenzip_cmd = str(_find_7z_by_registry())
	sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)		sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)

	if not relative_to is None and (output_dir / relative_to).exists():		if not relative_to is None and (output_dir / relative_to).exists():
	get_logger().error('Temporary unpacking directory already exists: %s',		get_logger().error('Temporary unpacking directory already exists: %s',
	output_dir / relative_to)		output_dir / relative_to)
	raise Exception()		raise FileExistsError()
	cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(output_dir)))		cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', f'-o{str(output_dir)}')
	get_logger().debug('7z command line: %s', ' '.join(cmd))		get_logger().debug('7z command line: %s', ' '.join(cmd))

	result = subprocess.run(cmd, check=False)		result = subprocess.run(cmd, check=False)
	if result.returncode != 0:		if result.returncode != 0:
	get_logger().error('7z command returned %s', result.returncode)		get_logger().error('7z command returned %s', result.returncode)
	raise Exception()		raise ChildProcessError()

	_process_relative_to(output_dir, relative_to)		_process_relative_to(output_dir, relative_to)

	@@ -296,20 +297,20 @@ def extract_with_winrar(archive_path, output_dir, relative_to, extractors=None):
	if winrar_cmd == USE_REGISTRY:		if winrar_cmd == USE_REGISTRY:
	if not get_running_platform() == PlatformEnum.WINDOWS:		if not get_running_platform() == PlatformEnum.WINDOWS:
	get_logger().error('"%s" for WinRAR is only available on Windows', winrar_cmd)		get_logger().error('"%s" for WinRAR is only available on Windows', winrar_cmd)
	raise Exception()		raise EnvironmentError()
	winrar_cmd = str(_find_winrar_by_registry())		winrar_cmd = str(_find_winrar_by_registry())
	winrar_bin = _find_extractor_by_cmd(winrar_cmd)		winrar_bin = _find_extractor_by_cmd(winrar_cmd)

	if not relative_to is None and (output_dir / relative_to).exists():		if not relative_to is None and (output_dir / relative_to).exists():
	get_logger().error('Temporary unpacking directory already exists: %s',		get_logger().error('Temporary unpacking directory already exists: %s',
	output_dir / relative_to)		output_dir / relative_to)
	raise Exception()		raise FileExistsError()
	cmd = (winrar_bin, 'x', '-o+', str(archive_path), str(output_dir))		cmd = (winrar_bin, 'x', '-o+', str(archive_path), str(output_dir))
	get_logger().debug('WinRAR command line: %s', ' '.join(cmd))		get_logger().debug('WinRAR command line: %s', ' '.join(cmd))

	result = subprocess.run(cmd, check=False)		result = subprocess.run(cmd, check=False)
	if result.returncode != 0:		if result.returncode != 0:
	get_logger().error('WinRAR command returned %s', result.returncode)		get_logger().error('WinRAR command returned %s', result.returncode)
	raise Exception()		raise ChildProcessError()

	_process_relative_to(output_dir, relative_to)		_process_relative_to(output_dir, relative_to)

domain_substitution/domain_substitution.py

+8 −8

Original line number	Original line	Diff line number	Diff line
	@@ -106,7 +106,7 @@ def _substitute_path(path, regex_iter):
	except UnicodeDecodeError:		except UnicodeDecodeError:
	continue		continue
	if not content:		if not content:
	raise UnicodeDecodeError('Unable to decode with any encoding: %s' % path)		raise UnicodeDecodeError(f'Unable to decode with any encoding: {path}')
	file_subs = 0		file_subs = 0
	for regex_pair in regex_iter:		for regex_pair in regex_iter:
	content, sub_count = regex_pair.pattern.subn(regex_pair.replacement, content)		content, sub_count = regex_pair.pattern.subn(regex_pair.replacement, content)
	@@ -206,17 +206,17 @@ def apply_substitution(regex_path, files_path, source_tree, domainsub_cache):
	resolved_tree = source_tree.resolve()		resolved_tree = source_tree.resolve()
	regex_pairs = DomainRegexList(regex_path).regex_pairs		regex_pairs = DomainRegexList(regex_path).regex_pairs
	fileindex_content = io.BytesIO()		fileindex_content = io.BytesIO()
	with tarfile.open(str(domainsub_cache), 'w:%s' % domainsub_cache.suffix[1:],		with tarfile.open(str(domainsub_cache), f'w:{domainsub_cache.suffix[1:]}',
	compresslevel=1) if domainsub_cache else open(os.devnull, 'w') as cache_tar:		compresslevel=1) if domainsub_cache else open(
			os.devnull, 'w', encoding=ENCODING) as cache_tar:
	for relative_path in filter(len, files_path.read_text().splitlines()):		for relative_path in filter(len, files_path.read_text().splitlines()):
	if _INDEX_HASH_DELIMITER in relative_path:		if _INDEX_HASH_DELIMITER in relative_path:
	if domainsub_cache:		if domainsub_cache:
	# Cache tar will be incomplete; remove it for convenience		# Cache tar will be incomplete; remove it for convenience
	cache_tar.close()		cache_tar.close()
	domainsub_cache.unlink()		domainsub_cache.unlink()
	raise ValueError(		raise ValueError(f'Path "{relative_path}" contains '
	'Path "%s" contains the file index hash delimiter "%s"' % relative_path,		f'the file index hash delimiter "{_INDEX_HASH_DELIMITER}"')
	_INDEX_HASH_DELIMITER)
	path = resolved_tree / relative_path		path = resolved_tree / relative_path
	if not path.exists():		if not path.exists():
	get_logger().warning('Skipping non-existant path: %s', path)		get_logger().warning('Skipping non-existant path: %s', path)
	@@ -230,8 +230,8 @@ def apply_substitution(regex_path, files_path, source_tree, domainsub_cache):
	get_logger().info('Path has no substitutions: %s', relative_path)		get_logger().info('Path has no substitutions: %s', relative_path)
	continue		continue
	if domainsub_cache:		if domainsub_cache:
	fileindex_content.write('{}{}{:08x}\n'.format(relative_path, _INDEX_HASH_DELIMITER,		fileindex_content.write(
	crc32_hash).encode(ENCODING))		f'{relative_path}{_INDEX_HASH_DELIMITER}{crc32_hash:08x}\n'.encode(ENCODING))
	orig_tarinfo = tarfile.TarInfo(str(Path(_ORIG_DIR) / relative_path))		orig_tarinfo = tarfile.TarInfo(str(Path(_ORIG_DIR) / relative_path))
	orig_tarinfo.size = len(orig_content)		orig_tarinfo.size = len(orig_content)
	with io.BytesIO(orig_content) as orig_file:		with io.BytesIO(orig_content) as orig_file:

update_domain_substitution.sh

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -4,7 +4,7 @@ root_dir=$(dirname "$(readlink -f "$0")")

	cd $root_dir/domain_substitution		cd $root_dir/domain_substitution

	branch="138.0.7204.49-1"		branch="138.0.7204.157-1"
	if [ -d ungoogled-chromium ]; then		if [ -d ungoogled-chromium ]; then
	cd ungoogled-chromium		cd ungoogled-chromium
	git fetch origin $branch		git fetch origin $branch