Merge "Checkpoint new build orchestrator" am: a96be433c4 (c7de0493) · Commits · e / os / android_build

envsetup.sh

+45 −2

Original line number	Diff line number	Diff line
		@@ -456,7 +456,7 @@ function multitree_lunch()
		if $(echo "$1" \| grep -q '^-') ; then
		# Calls starting with a -- argument are passed directly and the function
		# returns with the lunch.py exit code.
		build/make/orchestrator/core/lunch.py "$@"
		build/build/make/orchestrator/core/lunch.py "$@"
		code=$?
		if [[ $code -eq 2 ]] ; then
		echo 1>&2
		@@ -467,7 +467,7 @@ function multitree_lunch()
		fi
		else
		# All other calls go through the --lunch variant of lunch.py
		results=($(build/make/orchestrator/core/lunch.py --lunch "$@"))
		results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@"))
		code=$?
		if [[ $code -eq 2 ]] ; then
		echo 1>&2
		@@ -954,6 +954,34 @@ function gettop
		fi
		}

		# TODO: Merge into gettop as part of launching multitree
		function multitree_gettop
		{
		local TOPFILE=build/build/make/core/envsetup.mk
		if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then
		# The following circumlocution ensures we remove symlinks from TOP.
		(cd "$TOP"; PWD= /bin/pwd)
		else
		if [ -f $TOPFILE ] ; then
		# The following circumlocution (repeated below as well) ensures
		# that we record the true directory name and not one that is
		# faked up with symlink names.
		PWD= /bin/pwd
		else
		local HERE=$PWD
		local T=
		while [ $ ! \( -f $TOPFILE $ \) -a $ "$PWD" != "/" $ ]; do
		\cd ..
		T=`PWD= /bin/pwd -P`
		done
		\cd "$HERE"
		if [ -f "$T/$TOPFILE" ]; then
		echo "$T"
		fi
		fi
		fi
		}

		function croot()
		{
		local T=$(gettop)
		@@ -1836,6 +1864,21 @@ function make()
		_wrap_build $(get_make_command "$@") "$@"
		}

		function _multitree_lunch_error()
		{
		>&2 echo "Couldn't locate the top of the tree. Please run \'source build/envsetup.sh\' and multitree_lunch from the root of your workspace."
		}

		function multitree_build()
		{
		if T="$(multitree_gettop)"; then
		"$T/build/build/orchestrator/core/orchestrator.py" "$@"
		else
		_multitree_lunch_error
		return 1
		fi
		}

		function provision()
		{
		if [ ! "$ANDROID_PRODUCT_OUT" ]; then

orchestrator/README

0 → 100644

+7 −0

Original line number	Diff line number	Diff line
		DEMO

		from the root of the workspace

		ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build
		ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build

orchestrator/core/api_assembly.py

0 → 100644

+151 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/python3
		#
		# Copyright (C) 2022 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import json
		import os

		def assemble_apis(inner_trees):

		# Find all of the contributions from the inner tree
		contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)

		# Load and validate the contribution files
		# TODO: Check timestamps and skip unnecessary work
		contributions = []
		for tree_key, filenames in contribution_files_dict.items():
		for filename in filenames:
		contribution_data = load_contribution_file(filename)
		if not contribution_data:
		continue
		# TODO: Validate the configs, especially that the domains match what we asked for
		# from the lunch config.
		contributions.append(contribution_data)

		# Group contributions by language and API surface
		stub_libraries = collate_contributions(contributions)

		# Iterate through all of the stub libraries and generate rules to assemble them
		# and Android.bp/BUILD files to make those available to inner trees.
		# TODO: Parallelize? Skip unnecessary work?
		ninja_file = NinjaFile() # TODO: parameters?
		build_file = BuildFile() # TODO: parameters?
		for stub_library in stub_libraries:
		STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)

		# TODO: Handle host_executables separately or as a StubLibrary language?


		def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
		"Scan an inner_tree's out dir for the api contribution files."
		directory = inner_tree.out.api_contributions_dir()
		result = []
		with os.scandir(directory) as it:
		for dirent in it:
		if not dirent.is_file():
		break
		if dirent.name.endswith(".json"):
		result.append(os.path.join(directory, dirent.name))
		return result


		def load_contribution_file(filename):
		"Load and return the API contribution at filename. On error report error and return None."
		with open(filename) as f:
		try:
		return json.load(f)
		except json.decoder.JSONDecodeError as ex:
		# TODO: Error reporting
		raise ex


		class StubLibraryContribution(object):
		def __init__(self, api_domain, library_contribution):
		self.api_domain = api_domain
		self.library_contribution = library_contribution


		class StubLibrary(object):
		def __init__(self, language, api_surface, api_surface_version, name):
		self.language = language
		self.api_surface = api_surface
		self.api_surface_version = api_surface_version
		self.name = name
		self.contributions = []

		def add_contribution(self, contrib):
		self.contributions.append(contrib)


		def collate_contributions(contributions):
		"""Take the list of parsed API contribution files, and group targets by API Surface, version,
		language and library name, and return a StubLibrary object for each of those.
		"""
		grouped = {}
		for contribution in contributions:
		for language in STUB_LANGUAGE_HANDLERS.keys():
		for library in contribution.get(language, []):
		key = (language, contribution["name"], contribution["version"], library["name"])
		stub_library = grouped.get(key)
		if not stub_library:
		stub_library = StubLibrary(language, contribution["name"],
		contribution["version"], library["name"])
		grouped[key] = stub_library
		stub_library.add_contribution(StubLibraryContribution(
		contribution["api_domain"], library))
		return list(grouped.values())


		def assemble_cc_api_library(ninja_file, build_file, stub_library):
		print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
		stub_library.name))
		for contrib in stub_library.contributions:
		print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
		# TODO: Implement me


		def assemble_java_api_library(ninja_file, build_file, stub_library):
		print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
		stub_library.name))
		for contrib in stub_library.contributions:
		print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
		# TODO: Implement me


		def assemble_resource_api_library(ninja_file, build_file, stub_library):
		print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
		stub_library.name))
		for contrib in stub_library.contributions:
		print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
		# TODO: Implement me


		STUB_LANGUAGE_HANDLERS = {
		"cc_libraries": assemble_cc_api_library,
		"java_libraries": assemble_java_api_library,
		"resource_libraries": assemble_resource_api_library,
		}


		class NinjaFile(object):
		"Generator for build actions and dependencies."
		pass


		class BuildFile(object):
		"Abstract generator for Android.bp files and BUILD files."
		pass

orchestrator/core/api_domain.py

0 → 100644

+28 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/python3
		#
		# Copyright (C) 2022 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		class ApiDomain(object):
		def __init__(self, name, tree, product):
		# Product will be null for modules
		self.name = name
		self.tree = tree
		self.product = product

		def __str__(self):
		return "ApiDomain(name=\"%s\" tree.root=\"%s\" product=%s)" % (
		self.name, self.tree.root,
		"None" if self.product is None else "\"%s\"" % self.product)

orchestrator/core/api_export.py

0 → 100644

+20 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/python3
		#
		# Copyright (C) 2022 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		def export_apis_from_tree(tree_key, inner_tree, cookie):
		inner_tree.invoke(["export_api_contributions"])