Merge changes I4a43604b,I7200f5f9,If3402a0a (55c090eb) · Commits · e / os / android_build

core/Makefile

+11 −0

Original line number	Diff line number	Diff line
		@@ -495,9 +495,20 @@ $(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
		--title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \
		--codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \
		--out_dir="$(OUT_DIR)" \
		--mode=html \
		> $@
		$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML))

		MK2BP_REMAINING_CSV := $(PRODUCT_OUT)/mk2bp_remaining.csv
		$(MK2BP_REMAINING_CSV): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT)
		@rm -f $@
		$(hide) $(MK2BP_CATALOG_SCRIPT) \
		--device=$(TARGET_DEVICE) \
		--out_dir="$(OUT_DIR)" \
		--mode=csv \
		> $@
		$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_CSV))

		# -----------------------------------------------------------------
		# Modules use -Wno-error, or added default -Wall -Werror
		WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt

tools/mk2bp_catalog.py

+703 −560

Original line number	Diff line number	Diff line
		@@ -168,22 +168,24 @@ def is_google(dirname):
		return True
		return False

		def make_annotation_link(annotations, analysis, modules):
		if analysis:
		return "<a href='javascript:update_details(%d)'>%s</a>" % (
		annotations.Add(analysis, modules),
		len(analysis)
		)
		else:
		return "";


		def is_clean(makefile):
		for analysis in makefile.analyses.values():
		if analysis:
		return False
		return True

		def clean_and_only_blocked_by_clean(soong, all_makefiles, makefile):
		if not is_clean(makefile):
		return False
		modules = soong.reverse_makefiles[makefile.filename]
		for module in modules:
		for dep in soong.transitive_deps(module):
		for filename in soong.makefiles.get(dep, []):
		m = all_makefiles.get(filename)
		if m and not is_clean(m):
		return False
		return True

		class Annotations(object):
		def __init__(self):
		self.entries = []
		@@ -205,6 +207,7 @@ class SoongData(object):
		self.makefiles = dict()
		self.reverse_makefiles = dict()
		self.installed = dict()
		self.reverse_installed = dict()
		self.modules = set()

		for (module, module_type, problem, dependencies, makefiles, installed) in reader:
		@@ -222,6 +225,29 @@ class SoongData(object):
		self.reverse_makefiles.setdefault(f, []).append(module)
		for f in installed.strip().split(' '):
		self.installed[f] = module
		self.reverse_installed.setdefault(module, []).append(f)

		def transitive_deps(self, module):
		results = set()
		def traverse(module):
		for dep in self.deps.get(module, []):
		if not dep in results:
		results.add(dep)
		traverse(module)
		traverse(module)
		return results

		def contains_unblocked_modules(self, filename):
		for m in self.reverse_makefiles[filename]:
		if len(self.deps[m]) == 0:
		return True
		return False

		def contains_blocked_modules(self, filename):
		for m in self.reverse_makefiles[filename]:
		if len(self.deps[m]) > 0:
		return True
		return False

		def count_deps(depsdb, module, seen):
		"""Based on the depsdb, count the number of transitive dependencies.
		@@ -237,18 +263,6 @@ def count_deps(depsdb, module, seen):
		count += 1 + count_deps(depsdb, dep, seen)
		return count

		def contains_unblocked_modules(soong, modules):
		for m in modules:
		if len(soong.deps[m]) == 0:
		return True
		return False

		def contains_blocked_modules(soong, modules):
		for m in modules:
		if len(soong.deps[m]) > 0:
		return True
		return False

		OTHER_PARTITON = "_other"
		HOST_PARTITON = "_host"

		@@ -273,6 +287,27 @@ def format_module_link(module):
		def format_module_list(modules):
		return "".join(["<div>%s</div>" % format_module_link(m) for m in modules])

		def print_analysis_header(link, title):
		print("""
		<a name="%(link)s"></a>
		<h2>%(title)s</h2>
		<table>
		<tr>
		<th class="RowTitle">Directory</th>
		<th class="Count">Total</th>
		<th class="Count Clean">Easy</th>
		<th class="Count Clean">Unblocked Clean</th>
		<th class="Count Unblocked">Unblocked</th>
		<th class="Count Blocked">Blocked</th>
		<th class="Count Clean">Clean</th>
		""" % {
		"link": link,
		"title": title
		})
		for analyzer in ANALYZERS:
		print("""<th class="Count Warning">%s</th>""" % analyzer.title)
		print(" </tr>")

		def main():
		parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.")
		parser.add_argument("--device", type=str, required=True,
		@@ -287,6 +322,9 @@ def main():
		help="Equivalent of $OUT_DIR, which will also be checked if"
		+ " --out_dir is unset. If neither is set, default is"
		+ " 'out'.")
		parser.add_argument("--mode", type=str,
		default="html",
		help="output format: csv or html")

		args = parser.parse_args()

		@@ -297,14 +335,11 @@ def main():
		args.out_dir = args.out_dir[:-1]

		TARGET_DEVICE = args.device
		HOST_OUT_ROOT = args.out_dir + "host"
		global HOST_OUT_ROOT
		HOST_OUT_ROOT = args.out_dir + "/host"
		global PRODUCT_OUT
		PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE

		if args.title:
		page_title = args.title
		else:
		page_title = "Remaining Android.mk files"

		# Read target information
		# TODO: Pull from configurable location. This is also slightly different because it's
		# only a single build, where as the tree scanning we do below is all Android.mk files.
		@@ -312,10 +347,35 @@ def main():
		% PRODUCT_OUT, "r", errors="ignore") as csvfile:
		soong = SoongData(csv.reader(csvfile))

		# Read the makefiles
		all_makefiles = dict()
		for filename, modules in soong.reverse_makefiles.items():
		if filename.startswith(args.out_dir + "/"):
		continue
		all_makefiles[filename] = Makefile(filename)

		if args.mode == "html":
		HtmlProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()
		elif args.mode == "csv":
		CsvProcessor(args=args, soong=soong, all_makefiles=all_makefiles).execute()

		class HtmlProcessor(object):
		def __init__(self, args, soong, all_makefiles):
		self.args = args
		self.soong = soong
		self.all_makefiles = all_makefiles
		self.annotations = Annotations()

		def execute(self):
		if self.args.title:
		page_title = self.args.title
		else:
		page_title = "Remaining Android.mk files"

		# Which modules are installed where
		modules_by_partition = dict()
		partitions = set()
		for installed, module in soong.installed.items():
		for installed, module in self.soong.installed.items():
		partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed)
		modules_by_partition.setdefault(partition, []).append(module)
		partitions.add(partition)
		@@ -343,7 +403,7 @@ def main():
		overflow: hidden;
		}
		#tables {
		padding: 0 20px 0 20px;
		padding: 0 20px 40px 20px;
		overflow: scroll;
		flex: 2 2 600px;
		}
		@@ -359,7 +419,7 @@ def main():
		h2 {
		margin: 12px 0 4px 0;
		}
		.DirName {
		.RowTitle {
		text-align: left;
		width: 200px;
		min-width: 200px;
		@@ -395,6 +455,10 @@ def main():
		padding: 2px 4px;
		border-right: 2px solid white;
		}
		tr.TotalRow td {
		background-color: white;
		border-right-color: white;
		}
		tr.AospDir td {
		background-color: #e6f4ea;
		border-right-color: #e6f4ea;
		@@ -501,6 +565,7 @@ def main():

		print("""
		<span class='NavSpacer'></span><span class='NavSpacer'> </span>
		<a href='#summary'>Overall Summary</a>
		</div>
		<div id="container">
		<div id="tables">
		@@ -529,6 +594,16 @@ def main():
		<th>Total</th>
		<td>The total number of makefiles in this each directory.</td>
		</tr>
		<tr>
		<th class="Clean">Easy</th>
		<td>The number of makefiles that have no warnings themselves, and also
		none of their dependencies have warnings either.</td>
		</tr>
		<tr>
		<th class="Clean">Unblocked Clean</th>
		<td>The number of makefiles that are both Unblocked and Clean.</td>
		</tr>

		<tr>
		<th class="Unblocked">Unblocked</th>
		<td>Makefiles containing one or more modules that don't have any
		@@ -597,21 +672,22 @@ def main():
		</div>
		""")

		annotations = Annotations()
		overall_summary = Summary()

		# For each partition
		makefiles_for_partitions = dict()
		for partition in sorted(partitions):
		modules = modules_by_partition[partition]

		makefiles = set(itertools.chain.from_iterable(
		[soong.makefiles[module] for module in modules]))
		[self.soong.makefiles[module] for module in modules]))

		# Read makefiles
		summary = Summary()
		for filename in makefiles:
		if not filename.startswith(args.out_dir + "/"):
		summary.Add(Makefile(filename))
		makefile = self.all_makefiles.get(filename)
		if makefile:
		summary.Add(makefile)
		overall_summary.Add(makefile)

		# Categorize directories by who is responsible
		aosp_dirs = []
		@@ -625,66 +701,24 @@ def main():
		else:
		partner_dirs.append(dirname)

		print("""
		<a name="partition_%(partition)s"></a>
		<h2>%(partition)s</h2>
		<table>
		<tr>
		<th class="DirName">Directory</th>
		<th class="Count">Total</th>
		<th class="Count Unblocked">Unblocked</th>
		<th class="Count Blocked">Blocked</th>
		<th class="Count Clean">Clean</th>
		""" % {
		"partition": partition
		})

		for analyzer in ANALYZERS:
		print("""<th class="Count Warning">%s</th>""" % analyzer.title)
		print_analysis_header("partition_" + partition, partition)

		print(" </tr>")
		for dirgroup, rowclass in [(aosp_dirs, "AospDir"),
		(google_dirs, "GoogleDir"),
		(partner_dirs, "PartnerDir"),]:
		for dirname in dirgroup:
		makefiles = summary.directories[dirname]

		all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
		clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if is_clean(makefile)]
		unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if contains_unblocked_modules(soong,
		soong.reverse_makefiles[makefile.filename])]
		blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if contains_blocked_modules(soong,
		soong.reverse_makefiles[makefile.filename])]
		self.print_analysis_row(summary, modules,
		dirname, rowclass, summary.directories[dirname])

		print("""
		<tr class="%(rowclass)s">
		<td class="DirName">%(dirname)s</td>
		<td class="Count">%(makefiles)s</td>
		<td class="Count">%(unblocked)s</td>
		<td class="Count">%(blocked)s</td>
		<td class="Count">%(clean)s</td>
		""" % {
		"rowclass": rowclass,
		"dirname": dirname,
		"makefiles": make_annotation_link(annotations, all_makefiles, modules),
		"unblocked": make_annotation_link(annotations, unblocked_makefiles, modules),
		"blocked": make_annotation_link(annotations, blocked_makefiles, modules),
		"clean": make_annotation_link(annotations, clean_makefiles, modules),
		})
		for analyzer in ANALYZERS:
		analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
		print("""<td class="Count">%s</td>"""
		% make_annotation_link(annotations, analyses, modules))

		print(" </tr>")
		self.print_analysis_row(summary, modules,
		"Total", "TotalRow",
		set(itertools.chain.from_iterable(summary.directories.values())))
		print("""
		</table>
		""")

		module_details = [(count_deps(soong.deps, m, []), -count_deps(soong.reverse_deps, m, []), m)
		module_details = [(count_deps(self.soong.deps, m, []),
		-count_deps(self.soong.reverse_deps, m, []), m)
		for m in modules]
		module_details.sort()
		module_details = [m[2] for m in module_details]
		@@ -699,7 +733,7 @@ def main():
		altRow = True
		for module in module_details:
		analyses = set()
		for filename in soong.makefiles[module]:
		for filename in self.soong.makefiles[module]:
		makefile = summary.makefiles.get(filename)
		if makefile:
		for analyzer, analysis in makefile.analyses.items():
		@@ -711,13 +745,23 @@ def main():
		print(" <td><a name='module_%s'></a>%s</td>" % (module, module))
		print(" <td class='AnalysisCol'>%s</td>" % " ".join(["<span class='Analysis'>%s</span>" % title
		for title in analyses]))
		print(" <td>%s</td>" % count_deps(soong.deps, module, []))
		print(" <td>%s</td>" % format_module_list(soong.deps.get(module, [])))
		print(" <td>%s</td>" % count_deps(soong.reverse_deps, module, []))
		print(" <td>%s</td>" % format_module_list(soong.reverse_deps.get(module, [])))
		print(" <td>%s</td>" % count_deps(self.soong.deps, module, []))
		print(" <td>%s</td>" % format_module_list(self.soong.deps.get(module, [])))
		print(" <td>%s</td>" % count_deps(self.soong.reverse_deps, module, []))
		print(" <td>%s</td>" % format_module_list(self.soong.reverse_deps.get(module, [])))
		print("</tr>")
		print("""</table>""")

		print_analysis_header("summary", "Overall Summary")

		modules = [module for installed, module in self.soong.installed.items()]
		self.print_analysis_row(overall_summary, modules,
		"All Makefiles", "TotalRow",
		set(itertools.chain.from_iterable(overall_summary.directories.values())))
		print("""
		</table>
		""")

		print("""
		<script type="text/javascript">
		function close_details() {
		@@ -843,16 +887,16 @@ def main():

		var ANALYSIS = [
		""" % {
		"codesearch": args.codesearch,
		"codesearch": self.args.codesearch,
		})
		for entry, mods in annotations.entries:
		for entry, mods in self.annotations.entries:
		print(" [")
		for analysis in entry:
		print(" new Analysis('%(filename)s', %(modules)s, [%(line_matches)s])," % {
		"filename": analysis.filename,
		#"modules": json.dumps([m for m in mods if m in filename in soong.makefiles[m]]),
		#"modules": json.dumps([m for m in mods if m in filename in self.soong.makefiles[m]]),
		"modules": json.dumps(
		[m for m in soong.reverse_makefiles[analysis.filename] if m in mods]),
		[m for m in self.soong.reverse_makefiles[analysis.filename] if m in mods]),
		"line_matches": ", ".join([
		"new LineMatch(%d, %s)" % (lineno, json.dumps(text))
		for lineno, text in analysis.line_matches]),
		@@ -862,10 +906,10 @@ def main():
		];
		var MODULE_DATA = {
		""")
		for module in soong.modules:
		for module in self.soong.modules:
		print(" '%(name)s': new Module(%(deps)s)," % {
		"name": module,
		"deps": json.dumps(soong.deps[module]),
		"deps": json.dumps(self.soong.deps[module]),
		})
		print("""
		};
		@@ -887,6 +931,105 @@ def main():
		</html>
		""")

		def traverse_ready_makefiles(self, summary, makefiles):
		return [Analysis(makefile.filename, []) for makefile in makefiles
		if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)]

		def print_analysis_row(self, summary, modules, rowtitle, rowclass, makefiles):
		all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles]
		clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if is_clean(makefile)]
		easy_makefiles = self.traverse_ready_makefiles(summary, makefiles)
		unblocked_clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if (self.soong.contains_unblocked_modules(makefile.filename)
		and is_clean(makefile))]
		unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if self.soong.contains_unblocked_modules(makefile.filename)]
		blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles
		if self.soong.contains_blocked_modules(makefile.filename)]

		print("""
		<tr class="%(rowclass)s">
		<td class="RowTitle">%(rowtitle)s</td>
		<td class="Count">%(makefiles)s</td>
		<td class="Count">%(easy)s</td>
		<td class="Count">%(unblocked_clean)s</td>
		<td class="Count">%(unblocked)s</td>
		<td class="Count">%(blocked)s</td>
		<td class="Count">%(clean)s</td>
		""" % {
		"rowclass": rowclass,
		"rowtitle": rowtitle,
		"makefiles": self.make_annotation_link(all_makefiles, modules),
		"unblocked": self.make_annotation_link(unblocked_makefiles, modules),
		"blocked": self.make_annotation_link(blocked_makefiles, modules),
		"clean": self.make_annotation_link(clean_makefiles, modules),
		"unblocked_clean": self.make_annotation_link(unblocked_clean_makefiles, modules),
		"easy": self.make_annotation_link(easy_makefiles, modules),
		})

		for analyzer in ANALYZERS:
		analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)]
		print("""<td class="Count">%s</td>"""
		% self.make_annotation_link(analyses, modules))

		print(" </tr>")

		def make_annotation_link(self, analysis, modules):
		if analysis:
		return "<a href='javascript:update_details(%d)'>%s</a>" % (
		self.annotations.Add(analysis, modules),
		len(analysis)
		)
		else:
		return "";

		class CsvProcessor(object):
		def __init__(self, args, soong, all_makefiles):
		self.args = args
		self.soong = soong
		self.all_makefiles = all_makefiles

		def execute(self):
		csvout = csv.writer(sys.stdout)

		# Title row
		row = ["Filename", "Module", "Partitions", "Easy", "Unblocked Clean", "Unblocked",
		"Blocked", "Clean"]
		for analyzer in ANALYZERS:
		row.append(analyzer.title)
		csvout.writerow(row)

		# Makefile & module data
		for filename in sorted(self.all_makefiles.keys()):
		makefile = self.all_makefiles[filename]
		for module in self.soong.reverse_makefiles[filename]:
		row = [filename, module]
		# Partitions
		row.append(";".join(sorted(set([get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT,
		installed)
		for installed
		in self.soong.reverse_installed.get(module, [])]))))
		# Easy
		row.append(1
		if clean_and_only_blocked_by_clean(self.soong, self.all_makefiles, makefile)
		else "")
		# Unblocked Clean
		row.append(1
		if (self.soong.contains_unblocked_modules(makefile.filename) and is_clean(makefile))
		else "")
		# Unblocked
		row.append(1 if self.soong.contains_unblocked_modules(makefile.filename) else "")
		# Blocked
		row.append(1 if self.soong.contains_blocked_modules(makefile.filename) else "")
		# Clean
		row.append(1 if is_clean(makefile) else "")
		# Analysis
		for analyzer in ANALYZERS:
		row.append(1 if makefile.analyses.get(analyzer) else "")
		# Write results
		csvout.writerow(row)

		if __name__ == "__main__":
		main()