Merge "Make file list by partitions for all lunch targets" (722d0a60) · Commits · e / os / android_build

core/Makefile

+12 −4

Original line number	Diff line number	Diff line
		@@ -641,26 +641,34 @@ $(SOONG_TO_CONVERT): $(SOONG_CONV_DATA) $(SOONG_TO_CONVERT_SCRIPT)
		$(hide) $(SOONG_TO_CONVERT_SCRIPT) $< >$@
		$(call dist-for-goals,droidcore-unbundled,$(SOONG_TO_CONVERT))

		$(PRODUCT_OUT)/product_packages.txt:
		@rm -f $@
		echo "" > $@
		$(foreach x,$(PRODUCT_PACKAGES),echo $(x) >> $@$(newline))

		MK2BP_CATALOG_SCRIPT := build/make/tools/mk2bp_catalog.py
		PRODUCT_PACKAGES_TXT := $(PRODUCT_OUT)/product_packages.txt
		MK2BP_REMAINING_HTML := $(PRODUCT_OUT)/mk2bp_remaining.html
		$(MK2BP_REMAINING_HTML): PRIVATE_CODE_SEARCH_BASE_URL := "https://cs.android.com/android/platform/superproject/+/master:"
		$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
		$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT)
		@rm -f $@
		$(hide) $(MK2BP_CATALOG_SCRIPT) \
		--device=$(TARGET_DEVICE) \
		--product-packages=$(PRODUCT_PACKAGES_TXT) \
		--title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \
		--codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \
		--out_dir="$(OUT_DIR)" \
		--out-dir="$(OUT_DIR)" \
		--mode=html \
		> $@
		$(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_HTML))

		MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv
		$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
		$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) $(PRODUCT_PACKAGES_TXT)
		@rm -f $@
		$(hide) $(MK2BP_CATALOG_SCRIPT) \
		--device=$(TARGET_DEVICE) \
		--out_dir="$(OUT_DIR)" \
		--product-packages=$(PRODUCT_PACKAGES_TXT) \
		--out-dir="$(OUT_DIR)" \
		--mode=csv \
		> $@
		$(call dist-for-goals,droidcore-unbundled,$(MK2BP_REMAINING_CSV))

tools/mk2bp_catalog.py

+32 −6

Original line number	Diff line number	Diff line
		@@ -308,19 +308,31 @@ def print_analysis_header(link, title):
		print("""<th class="Count Warning">%s</th>""" % analyzer.title)
		print(" </tr>")

		# get all modules in $(PRODUCT_PACKAGE) and the corresponding deps
		def get_module_product_packages_plus_deps(initial_modules, result, soong_data):
		for module in initial_modules:
		if module in result:
		continue
		result.add(module)
		if module in soong_data.deps:
		get_module_product_packages_plus_deps(soong_data.deps[module], result, soong_data)

		def main():
		parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.")
		parser.add_argument("--device", type=str, required=True,
		help="TARGET_DEVICE")
		parser.add_argument("--product-packages", type=argparse.FileType('r'),
		default=None,
		help="PRODUCT_PACKAGES")
		parser.add_argument("--title", type=str,
		help="page title")
		parser.add_argument("--codesearch", type=str,
		default="https://cs.android.com/android/platform/superproject/+/master:",
		help="page title")
		parser.add_argument("--out_dir", type=str,
		parser.add_argument("--out-dir", type=str,
		default=None,
		help="Equivalent of $OUT_DIR, which will also be checked if"
		+ " --out_dir is unset. If neither is set, default is"
		+ " --out-dir is unset. If neither is set, default is"
		+ " 'out'.")
		parser.add_argument("--mode", type=str,
		default="html",
		@@ -354,16 +366,25 @@ def main():
		continue
		all_makefiles[filename] = Makefile(filename)

		# Get all the modules in $(PRODUCT_PACKAGES) and the correspoding deps
		product_package_modules_plus_deps = set()
		if args.product_packages:
		product_package_top_modules = args.product_packages.read().strip().split('\n')
		get_module_product_packages_plus_deps(product_package_top_modules, product_package_modules_plus_deps, soong)

		if args.mode == "html":
		HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
		HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles,
		product_packages_modules=product_package_modules_plus_deps).execute()
		elif args.mode == "csv":
		CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
		CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles,
		product_packages_modules=product_package_modules_plus_deps).execute()

		class HtmlProcessor(object):
		def __init__(self, args, soong, all_makefiles):
		def __init__(self, args, soong, all_makefiles, product_packages_modules):
		self.args = args
		self.soong = soong
		self.all_makefiles = all_makefiles
		self.product_packages_modules = product_packages_modules
		self.annotations = Annotations()

		def execute(self):
		@@ -376,6 +397,8 @@ class HtmlProcessor(object):
		modules_by_partition = dict()
		partitions = set()
		for installed, module in self.soong.installed.items():
		if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules:
		continue
		partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed)
		modules_by_partition.setdefault(partition, []).append(module)
		partitions.add(partition)
		@@ -985,10 +1008,11 @@ class HtmlProcessor(object):
		return "";

		class CsvProcessor(object):
		def __init__(self, args, soong, all_makefiles):
		def __init__(self, args, soong, all_makefiles, product_packages_modules):
		self.args = args
		self.soong = soong
		self.all_makefiles = all_makefiles
		self.product_packages_modules = product_packages_modules

		def execute(self):
		csvout = csv.writer(sys.stdout)
		@@ -1004,6 +1028,8 @@ class CsvProcessor(object):
		for filename in sorted(self.all_makefiles.keys()):
		makefile = self.all_makefiles[filename]
		for module in self.soong.reverse_makefiles[filename]:
		if len(self.product_packages_modules) > 0 and module not in self.product_packages_modules:
		continue
		row = [filename, module]
		# Partitions
		row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT,

tools/mk2bp_partition.py

0 → 100644

+100 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		The complete list of the remaining Make files in each partition for all lunch targets

		How to run?
		python3 $(path-to-file)/mk2bp_partition.py
		"""

		from pathlib import Path

		import csv
		import datetime
		import os
		import shutil
		import subprocess
		import sys
		import time

		def get_top():
		path = '.'
		while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')):
		if os.path.abspath(path) == '/':
		sys.exit('Could not find android source tree root.')
		path = os.path.join(path, '..')
		return os.path.abspath(path)

		# get the values of a build variable
		def get_build_var(variable, product, build_variant):
		"""Returns the result of the shell command get_build_var."""
		env = {
		**os.environ,
		'TARGET_PRODUCT': product if product else '',
		'TARGET_BUILD_VARIANT': build_variant if build_variant else '',
		}
		return subprocess.run([
		'build/soong/soong_ui.bash',
		'--dumpvar-mode',
		variable
		], check=True, capture_output=True, env=env, text=True).stdout.strip()

		def get_make_file_partitions():
		lunch_targets = set(get_build_var("all_named_products", "", "").split())
		total_lunch_targets = len(lunch_targets)
		makefile_by_partition = dict()
		partitions = set()
		current_count = 0
		start_time = time.time()
		# cannot run command `m lunch_target`
		broken_targets = {"mainline_sdk", "ndk"}
		for lunch_target in sorted(lunch_targets):
		current_count += 1
		current_time = time.time()
		print (current_count, "/", total_lunch_targets, lunch_target, datetime.timedelta(seconds=current_time - start_time))
		if lunch_target in broken_targets:
		continue
		installed_product_out = get_build_var("PRODUCT_OUT", lunch_target, "userdebug")
		filename = os.path.join(installed_product_out, "mk2bp_remaining.csv")
		copy_filename = os.path.join(installed_product_out, lunch_target + "_mk2bp_remaining.csv")
		# only generate if not exists
		if not os.path.exists(copy_filename):
		bash_cmd = "bash build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=" + lunch_target
		bash_cmd += " TARGET_BUILD_VARIANT=userdebug " + filename
		subprocess.run(bash_cmd, shell=True, text=True, check=True, stdout=subprocess.DEVNULL)
		# generate a copied .csv file, to avoid possible overwritings
		with open(copy_filename, "w") as file:
		shutil.copyfile(filename, copy_filename)

		# open mk2bp_remaining.csv file
		with open(copy_filename, "r") as csvfile:
		reader = csv.reader(csvfile, delimiter=",", quotechar='"')
		# bypass the header row
		next(reader, None)
		for row in reader:
		# read partition information
		partition = row[2]
		makefile_by_partition.setdefault(partition, set()).add(row[0])
		partitions.add(partition)

		# write merged make file list for each partition into a csv file
		installed_path = Path(installed_product_out).parents[0].as_posix()
		csv_path = installed_path + "/mk2bp_partition.csv"
		with open(csv_path, "wt") as csvfile:
		writer = csv.writer(csvfile, delimiter=",")
		count_makefile = 0
		for partition in sorted(partitions):
		number_file = len(makefile_by_partition[partition])
		count_makefile += number_file
		writer.writerow([partition, number_file])
		for makefile in sorted(makefile_by_partition[partition]):
		writer.writerow([makefile])
		row = ["The total count of make files is ", count_makefile]
		writer.writerow(row)

		def main():
		os.chdir(get_top())
		get_make_file_partitions()

		if __name__ == "__main__":
		main()