Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e5ed3474 authored by Joe Onorato's avatar Joe Onorato
Browse files

Add new soongdbg command and a big json file full of soong debugging info. 

In order to use soongdbg, you must run analysis with GENERATE_SOONG_DEBUG=true set
in the environment.

Test: GENERATE_SOONG_DEBUG=true m nothing ; soongdbg ...
Change-Id: If43676fe2784f05cd87c0ecb4a46ab676b91023f
parent b2a56627

bin/soongdbg

0 → 100755
+313 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import argparse

import fnmatch

import json

import os

import pathlib

import types

import sys





class Graph:

    def __init__(self, modules):

        def get_or_make_node(dictionary, id, module):

            node = dictionary.get(id)

            if node:

                if module and not node.module:

                    node.module = module

                return node

            node = Node(id, module)

            dictionary[id] = node

            return node

        self.nodes = dict()

        for module in modules.values():

            node = get_or_make_node(self.nodes, module.id, module)

            for d in module.deps:

                dep = get_or_make_node(self.nodes, d.id, None)

                node.deps.add(dep)

                dep.rdeps.add(node)



    def find_paths(self, id1, id2):

        # Throws KeyError if one of the names isn't found

        def recurse(node1, node2, visited):

            result = set()

            for dep in node1.rdeps:

                if dep == node2:

                    result.add(node2)

                if dep not in visited:

                    visited.add(dep)

                    found = recurse(dep, node2, visited)

                    if found:

                        result |= found

                        result.add(dep)

            return result

        node1 = self.nodes[id1]

        node2 = self.nodes[id2]

        # Take either direction

        p = recurse(node1, node2, set())

        if p:

            p.add(node1)

            return p

        p = recurse(node2, node1, set())

        p.add(node2)

        return p





class Node:

    def __init__(self, id, module):

        self.id = id

        self.module = module

        self.deps = set()

        self.rdeps = set()





PROVIDERS = [

    "android/soong/java.JarJarProviderData",

    "android/soong/java.BaseJarJarProviderData",

]





def format_node_label(node):

    if not node.module:

        return node.id

    if node.module.debug:

        module_debug = f"<tr><td>{node.module.debug}</td></tr>"

    else:

        module_debug = ""



    result = (f"<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">"

            + f"<tr><td><b>{node.module.name}</b></td></tr>"

            + module_debug

            + f"<tr><td>{node.module.type}</td></tr>")

    for p in node.module.providers:

        if p.type in PROVIDERS:

            result += "<tr><td><font color=\"#666666\">" + format_provider(p) + "</font></td></tr>"

    result += "</table>>"

    return result





def format_source_pos(file, lineno):

    result = file

    if lineno:

        result += f":{lineno}"

    return result





STRIP_TYPE_PREFIXES = [

    "android/soong/",

    "github.com/google/",

]





def format_provider(provider):

    result = ""

    for prefix in STRIP_TYPE_PREFIXES:

        if provider.type.startswith(prefix):

            result = provider.type[len(prefix):]

            break

    if not result:

        result = provider.type

    if True and provider.debug:

        result += " (" + provider.debug + ")"

    return result





def load_soong_debug():

    # Read the json

    try:

        with open(SOONG_DEBUG_DATA_FILENAME) as f:

            info = json.load(f, object_hook=lambda d: types.SimpleNamespace(**d))

    except IOError:

        sys.stderr.write(f"error: Unable to open {SOONG_DEBUG_DATA_FILENAME}. Make sure you have"

                         + " built with GENERATE_SOONG_DEBUG.\n")

        sys.exit(1)



    # Construct IDs, which are name + variant if the

    name_counts = dict()

    for m in info.modules:

        name_counts[m.name] = name_counts.get(m.name, 0) + 1

    def get_id(m):

        result = m.name

        if name_counts[m.name] > 1 and m.variant:

            result += "@@" + m.variant

        return result

    for m in info.modules:

        m.id = get_id(m)

        for dep in m.deps:

            dep.id = get_id(dep)



    return info





def load_modules():

    info = load_soong_debug()



    # Filter out unnamed modules

    modules = dict()

    for m in info.modules:

        if not m.name:

            continue

        modules[m.id] = m



    return modules





def load_graph():

    modules=load_modules()

    return Graph(modules)





def module_selection_args(parser):

    parser.add_argument("modules", nargs="*",

                        help="Modules to match. Can be glob-style wildcards.")

    parser.add_argument("--provider", nargs="+",

                        help="Match the given providers.")

    parser.add_argument("--dep", nargs="+",

                        help="Match the given providers.")





def load_and_filter_modules(args):

    # Which modules are printed

    matchers = []

    if args.modules:

        matchers.append(lambda m: [True for pattern in args.modules

                                   if fnmatch.fnmatchcase(m.name, pattern)])

    if args.provider:

        matchers.append(lambda m: [True for pattern in args.provider

                                   if [True for p in m.providers if p.type.endswith(pattern)]])

    if args.dep:

        matchers.append(lambda m: [True for pattern in args.dep

                                   if [True for d in m.deps if d.id == pattern]])



    if not matchers:

        sys.stderr.write("error: At least one module matcher must be supplied\n")

        sys.exit(1)



    info = load_soong_debug()

    for m in sorted(info.modules, key=lambda m: (m.name, m.variant)):

        if len([matcher for matcher in matchers if matcher(m)]) == len(matchers):

            yield m





def print_nodes(nodes):

    print("digraph {")

    for node in nodes:

        print(f"\"{node.id}\"[label={format_node_label(node)}];")

        for dep in node.deps:

            if dep in nodes:

                print(f"\"{node.id}\" -> \"{dep.id}\";")

    print("}")





def get_deps(nodes, root):

    if root in nodes:

        return

    nodes.add(root)

    for dep in root.deps:

        get_deps(nodes, dep)





class BetweenCommand:

    help = "Print the module graph between two nodes."



    def args(self, parser):

        parser.add_argument("module", nargs=2,

                            help="The two modules")



    def run(self, args):

        graph = load_graph()

        print_nodes(graph.find_paths(args.module[0], args.module[1]))





class DepsCommand:

    help = "Print the module graph of dependencies of one or more modules"



    def args(self, parser):

        parser.add_argument("module", nargs="+",

                            help="Module to print dependencies of")



    def run(self, args):

        graph = load_graph()

        nodes = set()

        err = False

        for id in sys.argv[3:]:

            root = graph.nodes.get(id)

            if not root:

                sys.stderr.write(f"error: Can't find root: {id}\n")

                err = True

                continue

            get_deps(nodes, root)

        if err:

            sys.exit(1)

        print_nodes(nodes)





class IdCommand:

    help = "Print the id (name + variant) of matching modules"



    def args(self, parser):

        module_selection_args(parser)



    def run(self, args):

        for m in load_and_filter_modules(args):

            print(m.id)





class QueryCommand:

    help = "Query details about modules"



    def args(self, parser):

        module_selection_args(parser)



    def run(self, args):

        for m in load_and_filter_modules(args):

            print(m.id)

            print(f"    type:     {m.type}")

            print(f"    location: {format_source_pos(m.source_file, m.source_line)}")

            for p in m.providers:

                print(f"    provider: {format_provider(p)}")

            for d in m.deps:

                print(f"    dep:      {d.id}")





COMMANDS = {

    "between": BetweenCommand(),

    "deps": DepsCommand(),

    "id": IdCommand(),

    "query": QueryCommand(),

}





def assert_env(name):

    val = os.getenv(name)

    if not val:

        sys.stderr.write(f"{name} not set. please make sure you've run lunch.")

    return val



ANDROID_BUILD_TOP = assert_env("ANDROID_BUILD_TOP")



TARGET_PRODUCT = assert_env("TARGET_PRODUCT")

OUT_DIR = os.getenv("OUT_DIR")

if not OUT_DIR:

    OUT_DIR = "out"

if OUT_DIR[0] != "/":

    OUT_DIR = pathlib.Path(ANDROID_BUILD_TOP).joinpath(OUT_DIR)

SOONG_DEBUG_DATA_FILENAME = pathlib.Path(OUT_DIR).joinpath("soong/soong-debug-info.json")





def main():

    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers(required=True, dest="command")

    for name in sorted(COMMANDS.keys()):

        command = COMMANDS[name]

        subparser = subparsers.add_parser(name, help=command.help)

        command.args(subparser)

    args = parser.parse_args()

    COMMANDS[args.command].run(args)

    sys.exit(0)





if __name__ == "__main__":

    main()

cmd/soong_build/main.go

+1 −0
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ func init() { 
	flag.BoolVar(&cmdlineArgs.MultitreeBuild, "multitree-build", false, "this is a multitree build")

	flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files")

	flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built")

	flag.StringVar(&cmdlineArgs.ModuleDebugFile, "soong_module_debug", "", "soong module debug info file to write")

	// Flags that probably shouldn't be flags of soong_build, but we haven't found

	// the time to remove them yet

	flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")

ui/build/config.go

+12 −0
Original line number Diff line number Diff line
@@ -115,6 +115,11 @@ type configImpl struct { 


	// Data source to write ninja weight list

	ninjaWeightListSource NinjaWeightListSource



	// This file is a detailed dump of all soong-defined modules for debugging purposes.

	// There's quite a bit of overlap with module-info.json and soong module graph. We

	// could consider merging them.

	moduleDebugFile string

}



type NinjaWeightListSource uint
@@ -273,6 +278,10 @@ func NewConfig(ctx Context, args ...string) Config { 
		ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")

	}



	if os.Getenv("GENERATE_SOONG_DEBUG") == "true" {

		ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json"))

	}



	ret.environ.Unset(

		// We're already using it

		"USE_SOONG_UI",
@@ -325,6 +334,9 @@ func NewConfig(ctx Context, args ...string) Config { 
		"ANDROID_DEV_SCRIPTS",

		"ANDROID_EMULATOR_PREBUILTS",

		"ANDROID_PRE_BUILD_PATHS",



		// We read it here already, don't let others share in the fun

		"GENERATE_SOONG_DEBUG",

	)



	if ret.UseGoma() || ret.ForceUseGoma() {

ui/build/soong.go

+5 −0
Original line number Diff line number Diff line
@@ -204,6 +204,11 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstra 
		commonArgs = append(commonArgs, "--build-from-source-stub")

	}



	if pb.config.moduleDebugFile != "" {

		commonArgs = append(commonArgs, "--soong_module_debug")

		commonArgs = append(commonArgs, pb.config.moduleDebugFile)

	}



	commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))

	invocationEnv := make(map[string]string)

	if pb.debugPort != "" {