Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0196f450 authored by Treehugger Robot's avatar Treehugger Robot Committed by Automerger Merge Worker
Browse files

Merge "Orchestrator can build end to end." am: d0035d5d am: 4d7877aa am: 19dcd91b 

Original change: https://android-review.googlesource.com/c/platform/build/+/2101130



Change-Id: I73b7d9fb4ab4cfe108dc126505397c2b0e3bf39d
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents 35280d81 19dcd91b

orchestrator/README

+3 −2
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ DEMO 


from the root of the workspace



ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build

ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build

multitree_lunch build/build/make/orchestrator/test_workspace/combo.mcombo eng



rm -rf out && multitree_build && echo "==== Files ====" && find out -type f

orchestrator/core/api_assembly.py

+3 −2
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ def assemble_apis(context, inner_trees): 
    contributions = []

    for tree_key, filenames in contribution_files_dict.items():

        for filename in filenames:

            json_data = load_contribution_file(filename)

            json_data = load_contribution_file(context, filename)

            if not json_data:

                continue

            # TODO: Validate the configs, especially that the domains match what we asked for
@@ -76,13 +76,14 @@ def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie): 
    return result





def load_contribution_file(filename):

def load_contribution_file(context, filename):

    "Load and return the API contribution at filename. On error report error and return None."

    with open(filename) as f:

        try:

            return json.load(f)

        except json.decoder.JSONDecodeError as ex:

            # TODO: Error reporting

            context.errors.error(ex.msg, filename, ex.lineno, ex.colno)

            raise ex

orchestrator/core/api_assembly_cc.py

+0 −7
Original line number Diff line number Diff line
@@ -17,17 +17,10 @@ 
import os



def assemble_cc_api_library(context, ninja, build_file, stub_library):

    print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,

        stub_library.api_surface_version, stub_library.name))

    for contrib in stub_library.contributions:

        print("  %s %s" % (contrib.api_domain, contrib.library_contribution))



    staging_dir = context.out.api_library_dir(stub_library.api_surface,

            stub_library.api_surface_version, stub_library.name)

    work_dir = context.out.api_library_work_dir(stub_library.api_surface,

            stub_library.api_surface_version, stub_library.name)

    print("staging_dir=%s" % (staging_dir))

    print("work_dir=%s" % (work_dir))



    # Generate rules to copy headers

    includes = []

orchestrator/core/final_packaging.py

+89 −1
Original line number Diff line number Diff line
@@ -13,10 +13,14 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import json

import os

import sys



import ninja_tools

import ninja_syntax # Has to be after ninja_tools because of the path hack



def final_packaging(context):

def final_packaging(context, inner_trees):

    """Pull together all of the previously defined rules into the final build stems."""



    with open(context.out.outer_ninja_file(), "w") as ninja_file:
@@ -25,5 +29,89 @@ def final_packaging(context): 
        # Add the api surfaces file

        ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))



        # For each inner tree

        for tree in inner_trees.keys():

            # TODO: Verify that inner_tree.ninja was generated



            # Read and verify file

            build_targets = read_build_targets_json(context, tree)

            if not build_targets:

                continue



            # Generate the ninja and build files for this inner tree

            generate_cross_domain_build_rules(context, ninja, tree, build_targets)



        # Finish writing the ninja file

        ninja.write()





def read_build_targets_json(context, tree):

    """Read and validate the build_targets.json file for the given tree."""

    try:

        f = open(tree.out.build_targets_file())

    except FileNotFoundError:

        # It's allowed not to have any artifacts (e.g. if a tree is a light tree with only APIs)

        return None



    data = None

    with f:

        try:

            data = json.load(f)

        except json.decoder.JSONDecodeError as ex:

            sys.stderr.write("Error parsing file: %s\n" % tree.out.build_targets_file())

            # TODO: Error reporting

            raise ex



    # TODO: Better error handling

    # TODO: Validate json schema

    return data





def generate_cross_domain_build_rules(context, ninja, tree, build_targets):

    "Generate the ninja and build files for the inner tree."

    # Include the inner tree's inner_tree.ninja

    ninja.add_subninja(ninja_syntax.Subninja(tree.out.main_ninja_file(), chDir=tree.root))



    # Generate module rules and files

    for module in build_targets.get("modules", []):

        generate_shared_module(context, ninja, tree, module)



    # Generate staging rules

    staging_dir = context.out.staging_dir()

    for staged in build_targets.get("staging", []):

        # TODO: Enforce that dest isn't in disallowed subdir of out or absolute

        dest = staged["dest"]

        dest = os.path.join(staging_dir, dest)

        if "src" in staged and "obj" in staged:

            context.errors.error("Can't have both \"src\" and \"obj\" tags in \"staging\" entry."

                    ) # TODO: Filename and line if possible

        if "src" in staged:

            ninja.add_copy_file(dest, os.path.join(tree.root, staged["src"]))

        elif "obj" in staged:

            ninja.add_copy_file(dest, os.path.join(tree.out.root(), staged["obj"]))

        ninja.add_global_phony("staging", [dest])



    # Generate dist rules

    dist_dir = context.out.dist_dir()

    for disted in build_targets.get("dist", []):

        # TODO: Enforce that dest absolute

        dest = disted["dest"]

        dest = os.path.join(dist_dir, dest)

        ninja.add_copy_file(dest, os.path.join(tree.root, disted["src"]))

        ninja.add_global_phony("dist", [dest])





def generate_shared_module(context, ninja, tree, module):

    """Generate ninja rules for the given build_targets.json defined module."""

    module_name = module["name"]

    module_type = module["type"]

    share_dir = context.out.module_share_dir(module_type, module_name)

    src_file = os.path.join(tree.root, module["file"])



    if module_type == "apex":

        ninja.add_copy_file(os.path.join(share_dir, module_name + ".apex"), src_file)

        # TODO: Generate build file



    else:

        # TODO: Better error handling

        raise Exception("Invalid module type: %s" % module)

orchestrator/core/inner_tree.py

+38 −7
Original line number Diff line number Diff line
@@ -36,23 +36,38 @@ class InnerTreeKey(object): 
    def __hash__(self):

        return hash((self.root, self.product))



    def _cmp(self, other):

        if self.root < other.root:

            return -1

        if self.root > other.root:

            return 1

        if self.product == other.product:

            return 0

        if self.product is None:

            return -1

        if other.product is None:

            return 1

        if self.product < other.product:

            return -1

        return 1



    def __eq__(self, other):

        return (self.root == other.root and self.product == other.product)

        return self._cmp(other) == 0



    def __ne__(self, other):

        return not self.__eq__(other)

        return self._cmp(other) != 0



    def __lt__(self, other):

        return (self.root, self.product) < (other.root, other.product)

        return self._cmp(other) < 0



    def __le__(self, other):

        return (self.root, self.product) <= (other.root, other.product)

        return self._cmp(other) <= 0



    def __gt__(self, other):

        return (self.root, self.product) > (other.root, other.product)

        return self._cmp(other) > 0



    def __ge__(self, other):

        return (self.root, self.product) >= (other.root, other.product)

        return self._cmp(other) >= 0





class InnerTree(object):
@@ -62,7 +77,12 @@ class InnerTree(object): 
        self.product = product

        self.domains = {}

        # TODO: Base directory on OUT_DIR

        self.out = OutDirLayout(context.out.inner_tree_dir(root))

        out_root = context.out.inner_tree_dir(root)

        if product:

            out_root += "_" + product

        else:

            out_root += "_unbundled"

        self.out = OutDirLayout(out_root)



    def __str__(self):

        return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@@ -138,6 +158,11 @@ class InnerTrees(object): 
        """Get an inner tree for tree_key"""

        return self.trees.get(tree_key)



    def keys(self):

        "Get the keys for the inner trees in name order."

        return [self.trees[k] for k in sorted(self.trees.keys())]





class OutDirLayout(object):

    """Encapsulates the logic about the layout of the inner tree out directories.

    See also context.OutDir for outer tree out dir contents."""
@@ -155,6 +180,12 @@ class OutDirLayout(object): 
    def api_contributions_dir(self):

        return os.path.join(self._root, "api_contributions")



    def build_targets_file(self):

        return os.path.join(self._root, "build_targets.json")



    def main_ninja_file(self):

        return os.path.join(self._root, "inner_tree.ninja")





def enquote(s):

    return "None" if s is None else "\"%s\"" % s