Merge "Generate ninja files for api assembly and run ninja." am: 9327b9cb4d (14c84a2d) · Commits · e / os / android_build

orchestrator/core/api_assembly.py

+43 −39

Original line number	Original line	Diff line number	Diff line
	@@ -14,11 +14,18 @@
	# See the License for the specific language governing permissions and		# See the License for the specific language governing permissions and
	# limitations under the License.		# limitations under the License.

			import collections
	import json		import json
	import os		import os
			import sys

	def assemble_apis(inner_trees):		import api_assembly_cc
			import ninja_tools


			ContributionData = collections.namedtuple("ContributionData", ("inner_tree", "json_data"))

			def assemble_apis(context, inner_trees):
	# Find all of the contributions from the inner tree		# Find all of the contributions from the inner tree
	contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)		contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)

	@@ -27,26 +34,34 @@ def assemble_apis(inner_trees):
	contributions = []		contributions = []
	for tree_key, filenames in contribution_files_dict.items():		for tree_key, filenames in contribution_files_dict.items():
	for filename in filenames:		for filename in filenames:
	contribution_data = load_contribution_file(filename)		json_data = load_contribution_file(filename)
	if not contribution_data:		if not json_data:
	continue		continue
	# TODO: Validate the configs, especially that the domains match what we asked for		# TODO: Validate the configs, especially that the domains match what we asked for
	# from the lunch config.		# from the lunch config.
	contributions.append(contribution_data)		contributions.append(ContributionData(inner_trees.get(tree_key), json_data))

	# Group contributions by language and API surface		# Group contributions by language and API surface
	stub_libraries = collate_contributions(contributions)		stub_libraries = collate_contributions(contributions)

			# Initialize the ninja file writer
			with open(context.out.api_ninja_file(), "w") as ninja_file:
			ninja = ninja_tools.Ninja(context, ninja_file)

			# Initialize the build file writer
			build_file = BuildFile() # TODO: parameters?

	# Iterate through all of the stub libraries and generate rules to assemble them		# Iterate through all of the stub libraries and generate rules to assemble them
	# and Android.bp/BUILD files to make those available to inner trees.		# and Android.bp/BUILD files to make those available to inner trees.
	# TODO: Parallelize? Skip unnecessary work?		# TODO: Parallelize? Skip unnecessary work?
	ninja_file = NinjaFile() # TODO: parameters?
	build_file = BuildFile() # TODO: parameters?
	for stub_library in stub_libraries:		for stub_library in stub_libraries:
	STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)		STUB_LANGUAGE_HANDLERS[stub_library.language](context, ninja, build_file, stub_library)

	# TODO: Handle host_executables separately or as a StubLibrary language?		# TODO: Handle host_executables separately or as a StubLibrary language?

			# Finish writing the ninja file
			ninja.write()


	def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):		def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
	"Scan an inner_tree's out dir for the api contribution files."		"Scan an inner_tree's out dir for the api contribution files."
	@@ -72,7 +87,8 @@ def load_contribution_file(filename):


	class StubLibraryContribution(object):		class StubLibraryContribution(object):
	def __init__(self, api_domain, library_contribution):		def __init__(self, inner_tree, api_domain, library_contribution):
			self.inner_tree = inner_tree
	self.api_domain = api_domain		self.api_domain = api_domain
	self.library_contribution = library_contribution		self.library_contribution = library_contribution

	@@ -96,54 +112,42 @@ def collate_contributions(contributions):
	grouped = {}		grouped = {}
	for contribution in contributions:		for contribution in contributions:
	for language in STUB_LANGUAGE_HANDLERS.keys():		for language in STUB_LANGUAGE_HANDLERS.keys():
	for library in contribution.get(language, []):		for library in contribution.json_data.get(language, []):
	key = (language, contribution["name"], contribution["version"], library["name"])		key = (language, contribution.json_data["name"],
			contribution.json_data["version"], library["name"])
	stub_library = grouped.get(key)		stub_library = grouped.get(key)
	if not stub_library:		if not stub_library:
	stub_library = StubLibrary(language, contribution["name"],		stub_library = StubLibrary(language, contribution.json_data["name"],
	contribution["version"], library["name"])		contribution.json_data["version"], library["name"])
	grouped[key] = stub_library		grouped[key] = stub_library
	stub_library.add_contribution(StubLibraryContribution(		stub_library.add_contribution(StubLibraryContribution(contribution.inner_tree,
	contribution["api_domain"], library))		contribution.json_data["api_domain"], library))
	return list(grouped.values())		return list(grouped.values())


	def assemble_cc_api_library(ninja_file, build_file, stub_library):		def assemble_java_api_library(context, ninja, build_file, stub_library):
	print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,		print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface,
	stub_library.name))		stub_library.api_surface_version, stub_library.name))
	for contrib in stub_library.contributions:		for contrib in stub_library.contributions:
	print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))		print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
	# TODO: Implement me		# TODO: Implement me


	def assemble_java_api_library(ninja_file, build_file, stub_library):		def assemble_resource_api_library(context, ninja, build_file, stub_library):
	print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,		print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface,
	stub_library.name))		stub_library.api_surface_version, stub_library.name))
	for contrib in stub_library.contributions:
	print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
	# TODO: Implement me


	def assemble_resource_api_library(ninja_file, build_file, stub_library):
	print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
	stub_library.name))
	for contrib in stub_library.contributions:		for contrib in stub_library.contributions:
	print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))		print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
	# TODO: Implement me		# TODO: Implement me


	STUB_LANGUAGE_HANDLERS = {		STUB_LANGUAGE_HANDLERS = {
	"cc_libraries": assemble_cc_api_library,		"cc_libraries": api_assembly_cc.assemble_cc_api_library,
	"java_libraries": assemble_java_api_library,		"java_libraries": assemble_java_api_library,
	"resource_libraries": assemble_resource_api_library,		"resource_libraries": assemble_resource_api_library,
	}		}


	class NinjaFile(object):
	"Generator for build actions and dependencies."
	pass


	class BuildFile(object):		class BuildFile(object):
	"Abstract generator for Android.bp files and BUILD files."		"Abstract generator for Android.bp files and BUILD files."
	pass		pass

orchestrator/core/api_assembly_cc.py

0 → 100644

+55 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/python3
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			import os

			def assemble_cc_api_library(context, ninja, build_file, stub_library):
			print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,
			stub_library.api_surface_version, stub_library.name))
			for contrib in stub_library.contributions:
			print(" %s %s" % (contrib.api_domain, contrib.library_contribution))

			staging_dir = context.out.api_library_dir(stub_library.api_surface,
			stub_library.api_surface_version, stub_library.name)
			work_dir = context.out.api_library_work_dir(stub_library.api_surface,
			stub_library.api_surface_version, stub_library.name)
			print("staging_dir=%s" % (staging_dir))
			print("work_dir=%s" % (work_dir))

			# Generate rules to copy headers
			includes = []
			include_dir = os.path.join(staging_dir, "include")
			for contrib in stub_library.contributions:
			for headers in contrib.library_contribution["headers"]:
			root = headers["root"]
			for file in headers["files"]:
			# TODO: Deal with collisions of the same name from multiple contributions
			include = os.path.join(include_dir, file)
			ninja.add_copy_file(include, os.path.join(contrib.inner_tree.root, root, file))
			includes.append(include)

			# Generate rule to run ndkstubgen


			# Generate rule to compile stubs to library

			# Generate phony rule to build the library
			# TODO: This name probably conflictgs with something
			ninja.add_phony("-".join((stub_library.api_surface, str(stub_library.api_surface_version),
			stub_library.name)), includes)

			# Generate build files

orchestrator/core/final_packaging.py

0 → 100644

+29 −0

Original line number	Original line	Diff line number	Diff line
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			import ninja_tools
			import ninja_syntax # Has to be after ninja_tools because of the path hack

			def final_packaging(context):
			"""Pull together all of the previously defined rules into the final build stems."""

			with open(context.out.outer_ninja_file(), "w") as ninja_file:
			ninja = ninja_tools.Ninja(context, ninja_file)

			# Add the api surfaces file
			ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))

			# Finish writing the ninja file
			ninja.write()

orchestrator/core/inner_tree.py

+9 −2

Original line number	Original line	Diff line number	Diff line
	@@ -56,13 +56,13 @@ class InnerTreeKey(object):


	class InnerTree(object):		class InnerTree(object):
	def __init__(self, root, product):		def __init__(self, context, root, product):
	"""Initialize with the inner tree root (relative to the workspace root)"""		"""Initialize with the inner tree root (relative to the workspace root)"""
	self.root = root		self.root = root
	self.product = product		self.product = product
	self.domains = {}		self.domains = {}
	# TODO: Base directory on OUT_DIR		# TODO: Base directory on OUT_DIR
	self.out = OutDirLayout(os.path.join("out", "trees", root))		self.out = OutDirLayout(context.out.inner_tree_dir(root))

	def __str__(self):		def __str__(self):
	return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),		return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
	@@ -134,7 +134,14 @@ class InnerTrees(object):
	return result		return result


			def get(self, tree_key):
			"""Get an inner tree for tree_key"""
			return self.trees.get(tree_key)

	class OutDirLayout(object):		class OutDirLayout(object):
			"""Encapsulates the logic about the layout of the inner tree out directories.
			See also context.OutDir for outer tree out dir contents."""

	def __init__(self, root):		def __init__(self, root):
	"Initialize with the root of the OUT_DIR for the inner tree."		"Initialize with the root of the OUT_DIR for the inner tree."
	self._root = root		self._root = root

orchestrator/core/ninja_runner.py

0 → 100644

+36 −0

Original line number	Original line	Diff line number	Diff line
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			import subprocess

			def run_ninja(context, targets):
			"""Run ninja.
			"""

			# Construct the command
			cmd = [
			context.tools.ninja(),
			"-f",
			context.out.outer_ninja_file(),
			] + targets

			# Run the command
			process = subprocess.run(cmd, shell=False)

			# TODO: Probably want better handling of inner tree failures
			if process.returncode:
			sys.stderr.write("Build error in outer tree.\nstopping multitree build.\n")
			sys.exit(1)