Update domain substitution (c8ae361d) · Commits · e / os / Browser

domain_substitution/_extraction.py

+11 −51

Original line number	Diff line number	Diff line
		@@ -14,7 +14,6 @@ import tarfile
		from pathlib import Path, PurePosixPath

		from _common import (USE_REGISTRY, PlatformEnum, ExtractorEnum, get_logger, get_running_platform)
		from prune_binaries import CONTINGENT_PATHS

		DEFAULT_EXTRACTORS = {
		ExtractorEnum.SEVENZIP: USE_REGISTRY,
		@@ -88,7 +87,7 @@ def _process_relative_to(unpack_root, relative_to):
		relative_root.rmdir()


		def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to):
		get_logger().debug('Using 7-zip extractor')
		if not relative_to is None and (output_dir / relative_to).exists():
		get_logger().error('Temporary unpacking directory already exists: %s',
		@@ -96,11 +95,6 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu
		raise Exception()
		cmd1 = (binary, 'x', str(archive_path), '-so')
		cmd2 = (binary, 'x', '-si', '-aoa', '-ttar', '-o{}'.format(str(output_dir)))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd2 += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('7z command line: %s \| %s', ' '.join(cmd1), ' '.join(cmd2))

		proc1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE)
		@@ -116,15 +110,10 @@ def _extract_tar_with_7z(binary, archive_path, output_dir, relative_to, skip_unu
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to):
		get_logger().debug('Using BSD or GNU tar extractor')
		output_dir.mkdir(exist_ok=True)
		cmd = (binary, '-xf', str(archive_path), '-C', str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('--exclude=%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('tar command line: %s', ' '.join(cmd))
		result = subprocess.run(cmd, check=False)
		if result.returncode != 0:
		@@ -136,15 +125,10 @@ def _extract_tar_with_tar(binary, archive_path, output_dir, relative_to, skip_un
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip_unused, sysroot):
		def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to):
		get_logger().debug('Using WinRAR extractor')
		output_dir.mkdir(exist_ok=True)
		cmd = (binary, 'x', '-o+', str(archive_path), str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/')), )
		get_logger().debug('WinRAR command line: %s', ' '.join(cmd))
		result = subprocess.run(cmd, check=False)
		if result.returncode != 0:
		@@ -154,7 +138,7 @@ def _extract_tar_with_winrar(binary, archive_path, output_dir, relative_to, skip
		_process_relative_to(output_dir, relative_to)


		def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot):
		def _extract_tar_with_python(archive_path, output_dir, relative_to):
		get_logger().debug('Using pure Python tar extractor')

		class NoAppendList(list):
		@@ -181,12 +165,6 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused,
		tar_file_obj.members = NoAppendList()
		for tarinfo in tar_file_obj:
		try:
		if skip_unused and [
		cpath for cpath in CONTINGENT_PATHS
		if tarinfo.name.startswith(str(relative_to) + '/' + cpath)
		and not (sysroot and f'{sysroot}-sysroot' in cpath)
		]:
		continue
		if relative_to is None:
		destination = output_dir / PurePosixPath(tarinfo.name)
		else:
		@@ -210,7 +188,7 @@ def _extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused,
		raise


		def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None):
		def extract_tar_file(archive_path, output_dir, relative_to, extractors=None):
		"""
		Extract regular or compressed tar archive into the output directory.

		@@ -233,8 +211,7 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot
		sevenzip_cmd = str(_find_7z_by_registry())
		sevenzip_bin = _find_extractor_by_cmd(sevenzip_cmd)
		if sevenzip_bin is not None:
		_extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		_extract_tar_with_7z(sevenzip_bin, archive_path, output_dir, relative_to)
		return

		# Use WinRAR if 7-zip is not found
		@@ -243,8 +220,7 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot
		winrar_cmd = str(_find_winrar_by_registry())
		winrar_bin = _find_extractor_by_cmd(winrar_cmd)
		if winrar_bin is not None:
		_extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		_extract_tar_with_winrar(winrar_bin, archive_path, output_dir, relative_to)
		return
		get_logger().warning(
		'Neither 7-zip nor WinRAR were found. Falling back to Python extractor...')
		@@ -252,17 +228,16 @@ def extract_tar_file(archive_path, output_dir, relative_to, skip_unused, sysroot
		# NOTE: 7-zip isn't an option because it doesn't preserve file permissions
		tar_bin = _find_extractor_by_cmd(extractors.get(ExtractorEnum.TAR))
		if not tar_bin is None:
		_extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to, skip_unused,
		sysroot)
		_extract_tar_with_tar(tar_bin, archive_path, output_dir, relative_to)
		return
		else:
		# This is not a normal code path, so make it clear.
		raise NotImplementedError(current_platform)
		# Fallback to Python-based extractor on all platforms
		_extract_tar_with_python(archive_path, output_dir, relative_to, skip_unused, sysroot)
		_extract_tar_with_python(archive_path, output_dir, relative_to)


		def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot, extractors=None):
		def extract_with_7z(archive_path, output_dir, relative_to, extractors=None):
		"""
		Extract archives with 7-zip into the output directory.
		Only supports archives with one layer of unpacking, so compressed tar archives don't work.
		@@ -292,11 +267,6 @@ def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot,
		output_dir / relative_to)
		raise Exception()
		cmd = (sevenzip_bin, 'x', str(archive_path), '-aoa', '-o{}'.format(str(output_dir)))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x!%s/%s' % (str(relative_to), cpath[:-1]), )
		get_logger().debug('7z command line: %s', ' '.join(cmd))

		result = subprocess.run(cmd, check=False)
		@@ -307,12 +277,7 @@ def extract_with_7z(archive_path, output_dir, relative_to, skip_unused, sysroot,
		_process_relative_to(output_dir, relative_to)


		def extract_with_winrar(archive_path,
		output_dir,
		relative_to,
		skip_unused,
		sysroot,
		extractors=None):
		def extract_with_winrar(archive_path, output_dir, relative_to, extractors=None):
		"""
		Extract archives with WinRAR into the output directory.
		Only supports archives with one layer of unpacking, so compressed tar archives don't work.
		@@ -340,11 +305,6 @@ def extract_with_winrar(archive_path,
		output_dir / relative_to)
		raise Exception()
		cmd = (winrar_bin, 'x', '-o+', str(archive_path), str(output_dir))
		if skip_unused:
		for cpath in CONTINGENT_PATHS:
		if sysroot and f'{sysroot}-sysroot' in cpath:
		continue
		cmd += ('-x%s%s%s' % (str(relative_to), os.sep, cpath[:-1].replace('/', os.sep)), )
		get_logger().debug('WinRAR command line: %s', ' '.join(cmd))

		result = subprocess.run(cmd, check=False)

domain_substitution/domain_regex.list

+1 −1

Original line number	Diff line number	Diff line
		fonts(\\?)\.googleapis(\\?)\.com#f0ntz\g<1>.9oo91e8p1\g<2>.qjz9zk
		google([A-Za-z\-]?\\?)\.com(?!mon)#9oo91e\g<1>.qjz9zk
		gstatic([A-Za-z\-]?\\?)\.com#95tat1c\g<1>.qjz9zk
		chrome([A-Za-z\-]?\\?)\.com#ch40me\g<1>.qjz9zk
		chrome([A-Za-z\-]?\\?)\.com(?!ponent)#ch40me\g<1>.qjz9zk
		chromium([A-Za-z\-]?\\?)\.org#ch40m1um\g<1>.qjz9zk
		mozilla([A-Za-z\-]?\\?)\.org#m0z111a\g<1>.qjz9zk
		facebook([A-Za-z\-]?\\?)\.com#f8c3b00k\g<1>.qjz9zk

domain_substitution/domain_substitution.list

+255 −476

File changed.

Preview size limit exceeded, changes collapsed.

domain_substitution/domain_substitution.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -284,7 +284,7 @@ def revert_substitution(domainsub_cache, source_tree):
		dir=str(resolved_tree)) as tmp_extract_name:
		extract_path = Path(tmp_extract_name)
		get_logger().debug('Extracting domain substitution cache...')
		extract_tar_file(domainsub_cache, extract_path, None, False, None)
		extract_tar_file(domainsub_cache, extract_path, None)

		# Validate source tree file hashes match
		get_logger().debug('Validating substituted files in source tree...')

domain_substitution/prune_binaries.py

+86 −10

Original line number	Diff line number	Diff line
		@@ -16,37 +16,97 @@ from pathlib import Path
		from _common import ENCODING, get_logger, add_common_params

		# List of paths to prune if they exist, excluded from domain_substitution and pruning lists
		# These allow the lists to be compatible between cloned and tarball sources
		# These allow the lists to be compatible between cloned, tarball, and lite tarball sources
		CONTINGENT_PATHS = (
		# Overridable git sources
		'third_party/angle/third_party/VK-GL-CTS/src/',
		# CIPD sources
		'buildtools/linux64/',
		'third_party/checkstyle/cipd/',
		'third_party/dawn/third_party/ninja/',
		'third_party/dawn/tools/golang/',
		'third_party/devtools-frontend/src/third_party/esbuild/',
		'third_party/google-java-format/',
		'third_party/libei/cipd/',
		'third_party/ninja/',
		'third_party/openscreen/src/third_party/ninja/',
		'third_party/siso/cipd/',
		'third_party/updater/chrome_linux64/',
		'third_party/updater/chrome_linux64_sans_iid/cipd/',
		'third_party/updater/chromium_linux64/',
		'third_party/updater/chromium_linux64_sans_iid/cipd/',
		'tools/luci-go/',
		'tools/resultdb/',
		'tools/skia_goldctl/linux/',
		# GCS sources
		'buildtools/linux64-format/',
		'third_party/blink/renderer/core/css/perftest_data/',
		'third_party/js_code_coverage/',
		'third_party/openscreen/src/buildtools/linux64-format/',
		'third_party/opus/tests/resources/',
		'third_party/subresource-filter-ruleset/data/',
		'tools/perf/page_sets/maps_perf_test/dataset/',
		# Sysroots(GCS), include all arches the clone script is able to obtain
		'build/linux/debian_bullseye_amd64-sysroot/',
		'build/linux/debian_bullseye_arm64-sysroot/',
		'build/linux/debian_bullseye_armhf-sysroot/',
		'build/linux/debian_bullseye_i386-sysroot/',
		'third_party/js_code_coverage/',
		'third_party/openscreen/src/buildtools/linux64/format/',
		'build/linux/debian_bullseye_mips64el-sysroot/',
		'build/linux/debian_bullseye_mipsel-sysroot/',
		# other
		'third_party/depot_tools/external_bin/',
		# Match removals for the tarball:
		# https://source.chromium.org/chromium/chromium/tools/build/+/main:recipes/recipe_modules/chromium/resources/export_tarball.py
		# nonessential
		'third_party/blink/tools/',
		'third_party/blink/web_tests/',
		'third_party/hunspell_dictionaries/',
		'third_party/hunspell/tests/',
		'third_party/jdk/current/',
		'third_party/jdk/extras/',
		'third_party/liblouis/src/tests/braille-specs/',
		'third_party/xdg-utils/tests/',
		'v8/test/',
		# test
		'base/tracing/test/data/',
		'chrome/test/data/',
		'components/test/data/',
		'content/test/data/accessibility/',
		'content/test/data/gpu/',
		'content/test/data/media/',
		'courgette/testdata/',
		'extensions/test/data/',
		'media/test/data/',
		'native_client/src/trusted/service_runtime/testdata/',
		'testing/libfuzzer/fuzzers/wasm_corpus/',
		'third_party/blink/perf_tests/',
		'third_party/breakpad/breakpad/src/processor/testdata/',
		'third_party/catapult/tracing/test_data/',
		'third_party/dawn/test/',
		'third_party/expat/src/testdata/',
		'third_party/harfbuzz-ng/src/test/',
		'third_party/llvm/llvm/test/',
		'third_party/ots/src/tests/fonts/',
		'third_party/rust-src/src/gcc/gcc/testsuite/',
		'third_party/rust-src/src/llvm-project/clang/test/',
		'third_party/rust-src/src/llvm-project/llvm/test/',
		'third_party/screen-ai/linux/resources/',
		'third_party/sqlite/src/test/',
		'third_party/swiftshader/tests/regres/',
		'third_party/test_fonts/test_fonts/',
		'tools/perf/testdata/',
		# lite tarball paths:
		# https://source.chromium.org/chromium/chromium/tools/build/+/main:recipes/recipes/publish_tarball.py
		'android_webview/',
		'buildtools/reclient/',
		'chrome/android/',
		'chromecast/',
		'ios/',
		'native_client/',
		'native_client_sdk/',
		'third_party/android_platform/',
		'third_party/angle/third_party/VK-GL-CTS/',
		'third_party/apache-linux/',
		'third_party/blink/manual_tests/',
		'third_party/blink/perf_tests/',
		'third_party/catapult/third_party/vinn/third_party/v8/',
		'third_party/closure_compiler/',
		'third_party/instrumented_libs/',
		'third_party/llvm/',
		'third_party/llvm-build/',
		@@ -55,8 +115,20 @@ CONTINGENT_PATHS = (
		'third_party/rust-src/',
		'third_party/rust-toolchain/',
		'third_party/webgl/',
		'third_party/blink/manual_tests/',
		'third_party/blink/perf_tests/',
		)

		# Files that should be excluded when pruning contingent paths.
		KEEP_FILES = (
		'chrome/test/data/webui/i18n_process_css_test.html',
		'chrome/test/data/webui/mojo/foobar.mojom',
		'v8/test/torque/test-torque.tq',
		)

		# File suffixes that should be excluded when pruning contingent paths.
		KEEP_SUFFIXES = ('.gn', '.gni', '.grd', '.grdp', '.isolate', '.pydeps')


		def prune_files(unpack_root, prune_list):
		"""
		@@ -80,13 +152,17 @@ def prune_files(unpack_root, prune_list):
		return unremovable_files


		def _prune_path(path):
		def _prune_path(path, unpack_root=None):
		"""
		Delete all files and directories in path.
		Delete files and directories in path.

		path is a pathlib.Path to the directory to be pruned
		unpack_root is a pathlib.Path to the source tree
		"""
		for node in sorted(path.rglob('*'), key=lambda l: len(str(l)), reverse=True):
		if unpack_root is not None and (node.suffix in KEEP_SUFFIXES
		or str(node.relative_to(unpack_root)) in KEEP_FILES):
		continue
		if node.is_file() or node.is_symlink():
		try:
		node.unlink()
		@@ -120,7 +196,7 @@ def prune_dirs(unpack_root, keep_contingent_paths, sysroot):
		get_logger().info('%s: %s', 'Exempt', cpath)
		continue
		get_logger().info('%s: %s', 'Exists' if Path(cpath).exists() else 'Absent', cpath)
		_prune_path(unpack_root / cpath)
		_prune_path(unpack_root / cpath, unpack_root)


		def _callback(args):

Original line number	Diff line number	Diff line
		fonts(\\?)\.googleapis(\\?)\.com#f0ntz\g<1>.9oo91e8p1\g<2>.qjz9zk
		google([A-Za-z\-]?\\?)\.com(?!mon)#9oo91e\g<1>.qjz9zk
		gstatic([A-Za-z\-]?\\?)\.com#95tat1c\g<1>.qjz9zk
		chrome([A-Za-z\-]?\\?)\.com#ch40me\g<1>.qjz9zk
		chrome([A-Za-z\-]?\\?)\.com(?!ponent)#ch40me\g<1>.qjz9zk
		chromium([A-Za-z\-]?\\?)\.org#ch40m1um\g<1>.qjz9zk
		mozilla([A-Za-z\-]?\\?)\.org#m0z111a\g<1>.qjz9zk
		facebook([A-Za-z\-]?\\?)\.com#f8c3b00k\g<1>.qjz9zk