Browser: Update domain_substitution (e5cd9e2f) · Commits · e / os / Browser

domain_substitution/.gitignore

+0 −1

Original line number	Diff line number	Diff line
		__pycache__/
		ungoogled-chromium/

domain_substitution/_extraction.py

+30 −21

Original line number	Diff line number	Diff line
		@@ -88,7 +88,7 @@ def _process_relative_to(unpack_root, relative_to):
		relative_root.rmdir()


		def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unused):
		def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		get_logger().debug('Using 7-zip extractor')
		if not relative_to is None and (output_dir / relative_to).exists():
		get_logger().error('Temporary unpacking directory already exists: %s',
		@@ -98,6 +98,8 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu
		cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(output_dir)))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd2 += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('7z command line: %s \| %s', ' '.join(cmd1), ' '.join(cmd2))

		@@ -114,12 +116,14 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_unused):
		def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		get_logger().debug('Using BSD or GNU tar extractor')
		output_dir.mkdir(exist_ok=True)
		cmd = (binary, '-xf', str(archive_path), '-C', str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('--exclude=%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('tar command line: %s', ' '.join(cmd))
		result = subprocess.run(cmd, check=False)
		@@ -132,12 +136,14 @@ def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_un
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip_unused):
		def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		get_logger().debug('Using WinRAR extractor')
		output_dir.mkdir(exist_ok=True)
		cmd = (binary, 'x', '-o+', str(archive_path), str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/')), )
		get_logger().debug('WinRAR command line: %s', ' '.join(cmd))
		result = subprocess.run(cmd, check=False)
		@@ -148,7 +154,7 @@ def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused):
		def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot):
		get_logger().debug('Using pure Python tar extractor')

		class NoAppendList(list):
		@@ -178,6 +184,7 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused)
		if skip_unused and [
		cpath for cpath in CONTINGENT_PATHS
		if tarinfo.name.startswith(str(relative_to) + '/' + cpath)
		and not (sysroot and f'{sysroot}-sysroot' in cpath)
		]:
		continue
		if relative_to is None:
		@@ -203,7 +210,7 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused)
		raise


		def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, extractors=None):
		def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None):
		"""
		Extract regular or compressed tar archive into the output directory.

		@@ -226,7 +233,8 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, extract
		sevenzip_cmd = str(_find_7z_by_registry())
		sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)
		if sevenzip_bin is not None:
		_extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to, skip_unused)
		_extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		return

		# Use WinRAR if 7-zip is not found
		@@ -235,7 +243,8 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, extract
		winrar_cmd = str(_find_winrar_by_registry())
		winrar_bin = _find_extractor_by_cmd(winrar_cmd)
		if winrar_bin is not None:
		_extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to, skip_unused)
		_extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		return
		get_logger().warning(
		'Neither 7-zip nor WinRAR were found. Falling back to Python extractor...')
		@@ -243,21 +252,17 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, extract
		# NOTE: 7-zip isn't an option because it doesn't preserve file permissions
		tar_bin = _find_extractor_by_cmd(extractors.get(ExtractorEnum.TAR))
		if not tar_bin is None:
		_extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to, skip_unused)
		_extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		return
		else:
		# This is not a normal code path, so make it clear.
		raise NotImplementedError(current_platform)
		# Fallback to Python-based extractor on all platforms
		_extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused)
		_extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot)


		def extract_with_7z(
		archive_path,
		output_dir,
		relative_to, #pylint: disable=too-many-arguments
		skip_unused,
		extractors=None):
		def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None):
		"""
		Extract archives with 7-zip into the output directory.
		Only supports archives with one layer of unpacking, so compressed tar archives don't work.
		@@ -289,6 +294,8 @@ def extract_with_7z(
		cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(output_dir)))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('7z command line: %s', ' '.join(cmd))

		@@ -300,11 +307,11 @@ def extract_with_7z(
		_process_relative_to(output_dir, relative_to)


		def extract_with_winrar(
		archive_path,
		def extract_with_winrar(archive_path,
		output_dir,
		relative_to, #pylint: disable=too-many-arguments
		relative_to,
		skip_unused,
		sysroot,
		extractors=None):
		"""
		Extract archives with WinRAR into the output directory.
		@@ -335,6 +342,8 @@ def extract_with_winrar(
		cmd = (winrar_bin, 'x', '-o+', str(archive_path), str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/', os.sep)), )
		get_logger().debug('WinRAR command line: %s', ' '.join(cmd))

domain_substitution/copy_domain_substitution.sh

+2 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,8 @@ fi

		grep -vFf domain_blacklist.list ungoogled-chromium/domain_substitution.list > domain_substitution.list
		cp -r ungoogled-chromium/domain_regex.list domain_regex.list
		cd ungoogled-chromium/utils
		cp _common.py domain_substitution.py _extraction.py prune_binaries.py ../../

		echo "Copy done"

domain_substitution/domain_substitution.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -284,7 +284,7 @@ def revert_substitution(domainsub_cache, source_tree):
		dir=str(resolved_tree)) as tmp_extract_name:
		extract_path = Path(tmp_extract_name)
		get_logger().debug('Extracting domain substitution cache...')
		extract_tar_file(domainsub_cache, extract_path, None, False)
		extract_tar_file(domainsub_cache, extract_path, None, False, None)

		# Validate source tree file hashes match
		get_logger().debug('Validating substituted files in source tree...')

domain_substitution/prune_binaries.py

+40 −16

Original line number	Diff line number	Diff line
		@@ -17,23 +17,16 @@ from _common import ENCODING, get_logger, add_common_params
		# List of paths to prune if they exist, excluded from domain_substitution and pruning lists
		# These allow the lists to be compatible between cloned and tarball sources
		CONTINGENT_PATHS = (
		# Sources
		# Overridable git sources
		'third_party/angle/third_party/VK-GL-CTS/src/',
		'third_party/llvm/',
		'third_party/rust-src/',
		# Binaries
		'third_party/instrumented_libs/',
		# CIPD sources
		'buildtools/linux64/',
		'buildtools/reclient/',
		'third_party/android_rust_toolchain/',
		'third_party/apache-linux/',
		'third_party/checkstyle/',
		'third_party/dawn/third_party/ninja/',
		'third_party/dawn/tools/golang/',
		'third_party/depot_tools/external_bin/',
		'third_party/devtools-frontend/src/third_party/esbuild/',
		'third_party/google-java-format/',
		'third_party/libei/',
		'third_party/llvm-build-tools/',
		'third_party/ninja/',
		'third_party/screen-ai/',
		'third_party/siso/',
		@@ -42,6 +35,21 @@ CONTINGENT_PATHS = (
		'tools/luci-go/',
		'tools/resultdb/',
		'tools/skia_goldctl/linux/',
		# GCS sources
		'base/tracing/test/data',
		'build/linux/debian_bullseye_amd64-sysroot/',
		'build/linux/debian_bullseye_i386-sysroot/',
		'buildtools/linux64-format/',
		'third_party/blink/renderer/core/css/perftest_data/',
		'third_party/js_code_coverage/',
		'third_party/llvm-build/Release+Asserts/',
		'third_party/node/linux/',
		'third_party/opus/tests/resources/',
		'third_party/rust-toolchain/',
		'third_party/subresource-filter-ruleset/data',
		'third_party/test_fonts/test_fonts',
		'third_party/tfhub_models/testdata/',
		'tools/perf/page_sets/maps_perf_test/dataset/',
		)


		@@ -88,16 +96,24 @@ def _prune_path(path):
		node.rmdir()


		def prune_dirs(unpack_root):
		def prune_dirs(unpack_root, keep_contingent_paths, sysroot):
		"""
		Delete all files and directories in pycache and CONTINGENT_PATHS directories.

		unpack_root is a pathlib.Path to the source tree
		keep_contingent_paths is a boolean that determines if the contingent paths should be pruned
		sysroot is a string that optionally defines a sysroot to exempt from pruning
		"""
		for pycache in unpack_root.rglob('__pycache__'):
		_prune_path(pycache)
		if keep_contingent_paths:
		get_logger().info('Keeping Contingent Paths')
		else:
		get_logger().info('Removing Contingent Paths')
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		get_logger().info('%s: %s', 'Exempt', cpath)
		continue
		get_logger().info('%s: %s', 'Exists' if Path(cpath).exists() else 'Absent', cpath)
		_prune_path(unpack_root / cpath)

		@@ -108,7 +124,7 @@ def _callback(args):
		sys.exit(1)
		if not args.pruning_list.exists():
		get_logger().error('Could not find the pruning list: %s', args.pruning_list)
		prune_dirs(args.directory)
		prune_dirs(args.directory, args.keep_contingent_paths, args.sysroot)
		prune_list = tuple(filter(len, args.pruning_list.read_text(encoding=ENCODING).splitlines()))
		unremovable_files = prune_files(args.directory, prune_list)
		if unremovable_files:
		@@ -123,6 +139,14 @@ def main():
		parser = argparse.ArgumentParser()
		parser.add_argument('directory', type=Path, help='The directory to apply binary pruning.')
		parser.add_argument('pruning_list', type=Path, help='Path to pruning.list')
		parser.add_argument('--keep-contingent-paths',
		action='store_true',
		help=('Skip pruning the contingent paths. '
		'Useful when building with the Google tooling is desired.'))
		parser.add_argument('--sysroot',
		choices=('amd64', 'i386'),
		help=('Skip pruning the sysroot for the specified architecture. '
		'Not needed when --keep-contingent-paths is used.'))
		add_common_params(parser)
		parser.set_defaults(callback=_callback)