Merge "Checkpoint new build orchestrator" (a96be433) · Commits · e / os / android_build

envsetup.sh

+45 −2

Original line number	Original line	Diff line number	Diff line
	@@ -456,7 +456,7 @@ function multitree_lunch()
	if $(echo "$1" \| grep -q '^-') ; then		if $(echo "$1" \| grep -q '^-') ; then
	# Calls starting with a -- argument are passed directly and the function		# Calls starting with a -- argument are passed directly and the function
	# returns with the lunch.py exit code.		# returns with the lunch.py exit code.
	build/make/orchestrator/core/lunch.py "$@"		build/build/make/orchestrator/core/lunch.py "$@"
	code=$?		code=$?
	if [[ $code -eq 2 ]] ; then		if [[ $code -eq 2 ]] ; then
	echo 1>&2		echo 1>&2
	@@ -467,7 +467,7 @@ function multitree_lunch()
	fi		fi
	else		else
	# All other calls go through the --lunch variant of lunch.py		# All other calls go through the --lunch variant of lunch.py
	results=($(build/make/orchestrator/core/lunch.py --lunch "$@"))		results=($(build/build/make/orchestrator/core/lunch.py --lunch "$@"))
	code=$?		code=$?
	if [[ $code -eq 2 ]] ; then		if [[ $code -eq 2 ]] ; then
	echo 1>&2		echo 1>&2
	@@ -944,6 +944,34 @@ function gettop
	fi		fi
	}		}

			# TODO: Merge into gettop as part of launching multitree
			function multitree_gettop
			{
			local TOPFILE=build/build/make/core/envsetup.mk
			if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then
			# The following circumlocution ensures we remove symlinks from TOP.
			(cd "$TOP"; PWD= /bin/pwd)
			else
			if [ -f $TOPFILE ] ; then
			# The following circumlocution (repeated below as well) ensures
			# that we record the true directory name and not one that is
			# faked up with symlink names.
			PWD= /bin/pwd
			else
			local HERE=$PWD
			local T=
			while [ $ ! \( -f $TOPFILE $ \) -a $ "$PWD" != "/" $ ]; do
			\cd ..
			T=`PWD= /bin/pwd -P`
			done
			\cd "$HERE"
			if [ -f "$T/$TOPFILE" ]; then
			echo "$T"
			fi
			fi
			fi
			}

	function croot()		function croot()
	{		{
	local T=$(gettop)		local T=$(gettop)
	@@ -1826,6 +1854,21 @@ function make()
	_wrap_build $(get_make_command "$@") "$@"		_wrap_build $(get_make_command "$@") "$@"
	}		}

			function _multitree_lunch_error()
			{
			>&2 echo "Couldn't locate the top of the tree. Please run \'source build/envsetup.sh\' and multitree_lunch from the root of your workspace."
			}

			function multitree_build()
			{
			if T="$(multitree_gettop)"; then
			"$T/build/build/orchestrator/core/orchestrator.py" "$@"
			else
			_multitree_lunch_error
			return 1
			fi
			}

	function provision()		function provision()
	{		{
	if [ ! "$ANDROID_PRODUCT_OUT" ]; then		if [ ! "$ANDROID_PRODUCT_OUT" ]; then

orchestrator/README

0 → 100644

+7 −0

Original line number	Original line	Diff line number	Diff line
			DEMO

			from the root of the workspace

			ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build
			ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build

orchestrator/core/api_assembly.py

0 → 100644

+151 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/python3
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			import json
			import os

			def assemble_apis(inner_trees):

			# Find all of the contributions from the inner tree
			contribution_files_dict = inner_trees.for_each_tree(api_contribution_files_for_inner_tree)

			# Load and validate the contribution files
			# TODO: Check timestamps and skip unnecessary work
			contributions = []
			for tree_key, filenames in contribution_files_dict.items():
			for filename in filenames:
			contribution_data = load_contribution_file(filename)
			if not contribution_data:
			continue
			# TODO: Validate the configs, especially that the domains match what we asked for
			# from the lunch config.
			contributions.append(contribution_data)

			# Group contributions by language and API surface
			stub_libraries = collate_contributions(contributions)

			# Iterate through all of the stub libraries and generate rules to assemble them
			# and Android.bp/BUILD files to make those available to inner trees.
			# TODO: Parallelize? Skip unnecessary work?
			ninja_file = NinjaFile() # TODO: parameters?
			build_file = BuildFile() # TODO: parameters?
			for stub_library in stub_libraries:
			STUB_LANGUAGE_HANDLERS[stub_library.language](ninja_file, build_file, stub_library)

			# TODO: Handle host_executables separately or as a StubLibrary language?


			def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie):
			"Scan an inner_tree's out dir for the api contribution files."
			directory = inner_tree.out.api_contributions_dir()
			result = []
			with os.scandir(directory) as it:
			for dirent in it:
			if not dirent.is_file():
			break
			if dirent.name.endswith(".json"):
			result.append(os.path.join(directory, dirent.name))
			return result


			def load_contribution_file(filename):
			"Load and return the API contribution at filename. On error report error and return None."
			with open(filename) as f:
			try:
			return json.load(f)
			except json.decoder.JSONDecodeError as ex:
			# TODO: Error reporting
			raise ex


			class StubLibraryContribution(object):
			def __init__(self, api_domain, library_contribution):
			self.api_domain = api_domain
			self.library_contribution = library_contribution


			class StubLibrary(object):
			def __init__(self, language, api_surface, api_surface_version, name):
			self.language = language
			self.api_surface = api_surface
			self.api_surface_version = api_surface_version
			self.name = name
			self.contributions = []

			def add_contribution(self, contrib):
			self.contributions.append(contrib)


			def collate_contributions(contributions):
			"""Take the list of parsed API contribution files, and group targets by API Surface, version,
			language and library name, and return a StubLibrary object for each of those.
			"""
			grouped = {}
			for contribution in contributions:
			for language in STUB_LANGUAGE_HANDLERS.keys():
			for library in contribution.get(language, []):
			key = (language, contribution["name"], contribution["version"], library["name"])
			stub_library = grouped.get(key)
			if not stub_library:
			stub_library = StubLibrary(language, contribution["name"],
			contribution["version"], library["name"])
			grouped[key] = stub_library
			stub_library.add_contribution(StubLibraryContribution(
			contribution["api_domain"], library))
			return list(grouped.values())


			def assemble_cc_api_library(ninja_file, build_file, stub_library):
			print("assembling cc_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
			stub_library.name))
			for contrib in stub_library.contributions:
			print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
			# TODO: Implement me


			def assemble_java_api_library(ninja_file, build_file, stub_library):
			print("assembling java_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
			stub_library.name))
			for contrib in stub_library.contributions:
			print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
			# TODO: Implement me


			def assemble_resource_api_library(ninja_file, build_file, stub_library):
			print("assembling resource_api_library %s-%s %s from:" % (stub_library.api_surface, stub_library.api_surface_version,
			stub_library.name))
			for contrib in stub_library.contributions:
			print(" %s %s" % (contrib.api_domain, contrib.library_contribution["api"]))
			# TODO: Implement me


			STUB_LANGUAGE_HANDLERS = {
			"cc_libraries": assemble_cc_api_library,
			"java_libraries": assemble_java_api_library,
			"resource_libraries": assemble_resource_api_library,
			}


			class NinjaFile(object):
			"Generator for build actions and dependencies."
			pass


			class BuildFile(object):
			"Abstract generator for Android.bp files and BUILD files."
			pass

orchestrator/core/api_domain.py

0 → 100644

+28 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/python3
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			class ApiDomain(object):
			def __init__(self, name, tree, product):
			# Product will be null for modules
			self.name = name
			self.tree = tree
			self.product = product

			def __str__(self):
			return "ApiDomain(name=\"%s\" tree.root=\"%s\" product=%s)" % (
			self.name, self.tree.root,
			"None" if self.product is None else "\"%s\"" % self.product)

orchestrator/core/api_export.py

0 → 100644

+20 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/python3
			#
			# Copyright (C) 2022 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.

			def export_apis_from_tree(tree_key, inner_tree, cookie):
			inner_tree.invoke(["export_api_contributions"])