Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c3589567 authored by Joe Onorato's avatar Joe Onorato
Browse files

Orchestrator can build end to end. 

This reduces the scope of the demo to just building and installing
a single .so, but it makes the demo actually build that single .so.

Next up, writing some unit tests and fleshing out functionality.

Test: see the README
Change-Id: I560904b786fbf69d3a83dbb08d496dba5a3192ca
parent 4117e785

orchestrator/README

+3 −2
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ DEMO 


from the root of the workspace



ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py master/.inner_build

ln -fs ../build/build/orchestrator/inner_build/inner_build_demo.py sc-mainline-prod/.inner_build

multitree_lunch build/build/make/orchestrator/test_workspace/combo.mcombo eng



rm -rf out && multitree_build && echo "==== Files ====" && find out -type f

orchestrator/core/api_assembly.py

+3 −2
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ def assemble_apis(context, inner_trees): 
    contributions = []

    for tree_key, filenames in contribution_files_dict.items():

        for filename in filenames:

            json_data = load_contribution_file(filename)

            json_data = load_contribution_file(context, filename)

            if not json_data:

                continue

            # TODO: Validate the configs, especially that the domains match what we asked for
@@ -76,13 +76,14 @@ def api_contribution_files_for_inner_tree(tree_key, inner_tree, cookie): 
    return result





def load_contribution_file(filename):

def load_contribution_file(context, filename):

    "Load and return the API contribution at filename. On error report error and return None."

    with open(filename) as f:

        try:

            return json.load(f)

        except json.decoder.JSONDecodeError as ex:

            # TODO: Error reporting

            context.errors.error(ex.msg, filename, ex.lineno, ex.colno)

            raise ex

orchestrator/core/api_assembly_cc.py

+0 −7
Original line number Diff line number Diff line
@@ -17,17 +17,10 @@ 
import os



def assemble_cc_api_library(context, ninja, build_file, stub_library):

    print("\nassembling cc_api_library %s-%s %s from:" % (stub_library.api_surface,

        stub_library.api_surface_version, stub_library.name))

    for contrib in stub_library.contributions:

        print("  %s %s" % (contrib.api_domain, contrib.library_contribution))



    staging_dir = context.out.api_library_dir(stub_library.api_surface,

            stub_library.api_surface_version, stub_library.name)

    work_dir = context.out.api_library_work_dir(stub_library.api_surface,

            stub_library.api_surface_version, stub_library.name)

    print("staging_dir=%s" % (staging_dir))

    print("work_dir=%s" % (work_dir))



    # Generate rules to copy headers

    includes = []

orchestrator/core/final_packaging.py

+89 −1
Original line number Diff line number Diff line
@@ -13,10 +13,14 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import json

import os

import sys



import ninja_tools

import ninja_syntax # Has to be after ninja_tools because of the path hack



def final_packaging(context):

def final_packaging(context, inner_trees):

    """Pull together all of the previously defined rules into the final build stems."""



    with open(context.out.outer_ninja_file(), "w") as ninja_file:
@@ -25,5 +29,89 @@ def final_packaging(context): 
        # Add the api surfaces file

        ninja.add_subninja(ninja_syntax.Subninja(context.out.api_ninja_file(), chDir=None))



        # For each inner tree

        for tree in inner_trees.keys():

            # TODO: Verify that inner_tree.ninja was generated



            # Read and verify file

            build_targets = read_build_targets_json(context, tree)

            if not build_targets:

                continue



            # Generate the ninja and build files for this inner tree

            generate_cross_domain_build_rules(context, ninja, tree, build_targets)



        # Finish writing the ninja file

        ninja.write()





def read_build_targets_json(context, tree):

    """Read and validate the build_targets.json file for the given tree."""

    try:

        f = open(tree.out.build_targets_file())

    except FileNotFoundError:

        # It's allowed not to have any artifacts (e.g. if a tree is a light tree with only APIs)

        return None



    data = None

    with f:

        try:

            data = json.load(f)

        except json.decoder.JSONDecodeError as ex:

            sys.stderr.write("Error parsing file: %s\n" % tree.out.build_targets_file())

            # TODO: Error reporting

            raise ex



    # TODO: Better error handling

    # TODO: Validate json schema

    return data





def generate_cross_domain_build_rules(context, ninja, tree, build_targets):

    "Generate the ninja and build files for the inner tree."

    # Include the inner tree's inner_tree.ninja

    ninja.add_subninja(ninja_syntax.Subninja(tree.out.main_ninja_file(), chDir=tree.root))



    # Generate module rules and files

    for module in build_targets.get("modules", []):

        generate_shared_module(context, ninja, tree, module)



    # Generate staging rules

    staging_dir = context.out.staging_dir()

    for staged in build_targets.get("staging", []):

        # TODO: Enforce that dest isn't in disallowed subdir of out or absolute

        dest = staged["dest"]

        dest = os.path.join(staging_dir, dest)

        if "src" in staged and "obj" in staged:

            context.errors.error("Can't have both \"src\" and \"obj\" tags in \"staging\" entry."

                    ) # TODO: Filename and line if possible

        if "src" in staged:

            ninja.add_copy_file(dest, os.path.join(tree.root, staged["src"]))

        elif "obj" in staged:

            ninja.add_copy_file(dest, os.path.join(tree.out.root(), staged["obj"]))

        ninja.add_global_phony("staging", [dest])



    # Generate dist rules

    dist_dir = context.out.dist_dir()

    for disted in build_targets.get("dist", []):

        # TODO: Enforce that dest absolute

        dest = disted["dest"]

        dest = os.path.join(dist_dir, dest)

        ninja.add_copy_file(dest, os.path.join(tree.root, disted["src"]))

        ninja.add_global_phony("dist", [dest])





def generate_shared_module(context, ninja, tree, module):

    """Generate ninja rules for the given build_targets.json defined module."""

    module_name = module["name"]

    module_type = module["type"]

    share_dir = context.out.module_share_dir(module_type, module_name)

    src_file = os.path.join(tree.root, module["file"])



    if module_type == "apex":

        ninja.add_copy_file(os.path.join(share_dir, module_name + ".apex"), src_file)

        # TODO: Generate build file



    else:

        # TODO: Better error handling

        raise Exception("Invalid module type: %s" % module)

orchestrator/core/inner_tree.py

+38 −7
Original line number Diff line number Diff line
@@ -36,23 +36,38 @@ class InnerTreeKey(object): 
    def __hash__(self):

        return hash((self.root, self.product))



    def _cmp(self, other):

        if self.root < other.root:

            return -1

        if self.root > other.root:

            return 1

        if self.product == other.product:

            return 0

        if self.product is None:

            return -1

        if other.product is None:

            return 1

        if self.product < other.product:

            return -1

        return 1



    def __eq__(self, other):

        return (self.root == other.root and self.product == other.product)

        return self._cmp(other) == 0



    def __ne__(self, other):

        return not self.__eq__(other)

        return self._cmp(other) != 0



    def __lt__(self, other):

        return (self.root, self.product) < (other.root, other.product)

        return self._cmp(other) < 0



    def __le__(self, other):

        return (self.root, self.product) <= (other.root, other.product)

        return self._cmp(other) <= 0



    def __gt__(self, other):

        return (self.root, self.product) > (other.root, other.product)

        return self._cmp(other) > 0



    def __ge__(self, other):

        return (self.root, self.product) >= (other.root, other.product)

        return self._cmp(other) >= 0





class InnerTree(object):
@@ -62,7 +77,12 @@ class InnerTree(object): 
        self.product = product

        self.domains = {}

        # TODO: Base directory on OUT_DIR

        self.out = OutDirLayout(context.out.inner_tree_dir(root))

        out_root = context.out.inner_tree_dir(root)

        if product:

            out_root += "_" + product

        else:

            out_root += "_unbundled"

        self.out = OutDirLayout(out_root)



    def __str__(self):

        return "InnerTree(root=%s product=%s domains=[%s])" % (enquote(self.root),
@@ -138,6 +158,11 @@ class InnerTrees(object): 
        """Get an inner tree for tree_key"""

        return self.trees.get(tree_key)



    def keys(self):

        "Get the keys for the inner trees in name order."

        return [self.trees[k] for k in sorted(self.trees.keys())]





class OutDirLayout(object):

    """Encapsulates the logic about the layout of the inner tree out directories.

    See also context.OutDir for outer tree out dir contents."""
@@ -155,6 +180,12 @@ class OutDirLayout(object): 
    def api_contributions_dir(self):

        return os.path.join(self._root, "api_contributions")



    def build_targets_file(self):

        return os.path.join(self._root, "build_targets.json")



    def main_ninja_file(self):

        return os.path.join(self._root, "inner_tree.ninja")





def enquote(s):

    return "None" if s is None else "\"%s\"" % s