Merge "Add script that dumps all product config varaibles." into main

# Copyright 2024, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Utilities for interacting with buildbot, with a simulation in a local environment"""

import os

import sys

# Check that the script is running from the root of the tree. Prevents subtle

# errors later, and CI always runs from the root of the tree.

if not os.path.exists("build/make/ci/buildbot.py"):

    raise Exception("CI script must be run from the root of the tree instead of: "

                    + os.getcwd())

# Check that we are using the hermetic interpreter

if "prebuilts/build-tools/" not in sys.executable:

    raise Exception("CI script must be run using the hermetic interpreter from "

                    + "prebuilts/build-tools instead of: " + sys.executable)

def OutDir():

    "Get the out directory. Will create it if needed."

    result = os.environ.get("OUT_DIR", "out")

    os.makedirs(result, exist_ok=True)

    return result

def DistDir():

    "Get the dist directory. Will create it if needed."

    result = os.environ.get("DIST_DIR", os.path.join(OutDir(), "dist"))

    os.makedirs(result, exist_ok=True)

    return result

#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B

# Copyright 2024, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Script to collect all of the make variables from all product config combos.

This script must be run from the root of the source tree.

See GetArgs() below or run dump_product_config for more information.

"""

import argparse

import asyncio

import contextlib

import csv

import dataclasses

import json

import multiprocessing

import os

import subprocess

import sys

import time

from typing import List, Dict, Tuple, Optional

import buildbot

# We have some BIG variables

csv.field_size_limit(sys.maxsize)

class DataclassJSONEncoder(json.JSONEncoder):

    """JSONEncoder for our custom types."""

    def default(self, o):

        if dataclasses.is_dataclass(o):

            return dataclasses.asdict(o)

        return super().default(o)

def GetProducts():

    """Get the all of the available TARGET_PRODUCT values."""

    try:

        stdout = subprocess.check_output(["build/soong/bin/list_products"], text=True)

    except subprocess.CalledProcessError:

        sys.exit(1)

    return [s.strip() for s in stdout.splitlines() if s.strip()]

def GetReleases(product):

    """For a given product, get the release configs available to it."""

    if True:

        # Hard code the list

        mainline_products = [

            "module_arm",

            "module_x86",

            "module_arm64",

            "module_riscv64",

            "module_x86_64",

            "module_arm64only",

            "module_x86_64only",

        ]

        if product in mainline_products:

            return ["trunk_staging", "trunk", "mainline"]

        else:

            return ["trunk_staging", "trunk", "next"]

    else:

        # Get it from the build system

        try:

            stdout = subprocess.check_output(["build/soong/bin/list_releases", product], text=True)

        except subprocess.CalledProcessError:

            sys.exit(1)

        return [s.strip() for s in stdout.splitlines() if s.strip()]

def GenerateAllLunchTargets():

    """Generate the full list of lunch targets."""

    for product in GetProducts():

        for release in GetReleases(product):

            for variant in ["user", "userdebug", "eng"]:

                yield (product, release, variant)

async def ParallelExec(parallelism, tasks):

    '''

    ParallelExec takes a parallelism number, and an iterator of tasks to run.

    Then it will run all the tasks, but a maximum of parallelism will be run at

    any given time. The tasks must be async functions that accept one argument,

    which will be an integer id of the worker that they're running on.

    '''

    tasks = iter(tasks)

    overall_start = time.monotonic()

    # lists so they can be modified from the inner function

    total_duration = [0]

    count = [0]

    async def dispatch(worker):

        while True:

            try:

                task = next(tasks)

                item_start = time.monotonic()

                await task(worker)

                now = time.monotonic()

                item_duration = now - item_start

                count[0] += 1

                total_duration[0] += item_duration

                sys.stderr.write(f"Timing: Items processed: {count[0]}, Wall time: {now-overall_start:0.1f} sec, Throughput: {(now-overall_start)/count[0]:0.3f} sec per item, Average duration: {total_duration[0]/count[0]:0.1f} sec\n")

            except StopIteration:

                return

    await asyncio.gather(*[dispatch(worker) for worker in range(parallelism)])

async def DumpProductConfigs(out, generator, out_dir):

    """Collects all of the product config data and store it in file."""

    # Write the outer json list by hand so we can stream it

    out.write("[")

    try:

        first_result = [True] # a list so it can be modified from the inner function

        def run(lunch):

            async def curried(worker):

                sys.stderr.write(f"running: {'-'.join(lunch)}\n")

                result = await DumpOneProductConfig(lunch, os.path.join(out_dir, f"lunchable_{worker}"))

                if first_result[0]:

                    out.write("\n")

                    first_result[0] = False

                else:

                    out.write(",\n")

                result.dumpToFile(out)

                sys.stderr.write(f"finished: {'-'.join(lunch)}\n")

            return curried

        await ParallelExec(multiprocessing.cpu_count(), (run(lunch) for lunch in generator))

    finally:

        # Close the json regardless of how we exit

        out.write("\n]\n")

@dataclasses.dataclass(frozen=True)

class Variable:

    """A variable name, value and where it was set."""

    name: str

    value: str

    location: str

@dataclasses.dataclass(frozen=True)

class ProductResult:

    product: str

    release: str

    variant: str

    board_includes: List[str]

    product_includes: Dict[str, List[str]]

    product_graph: List[Tuple[str, str]]

    board_vars: List[Variable]

    product_vars: List[Variable]

    def dumpToFile(self, f):

        json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)

@dataclasses.dataclass(frozen=True)

class ProductError:

    product: str

    release: str

    variant: str

    error: str

    def dumpToFile(self, f):

        json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)

def NormalizeInheritGraph(lists):

    """Flatten the inheritance graph to a simple list for easier querying."""

    result = set()

    for item in lists:

        for i in range(len(item)):

            result.add((item[i+1] if i < len(item)-1 else "", item[i]))

    return sorted(list(result))

def ParseDump(lunch, filename) -> ProductResult:

    """Parses the csv and returns a tuple of the data."""

    def diff(initial, final):

        return [after for after in final.values() if

                initial.get(after.name, Variable(after.name, "", "<unset>")).value != after.value]

    product_initial = {}

    product_final = {}

    board_initial = {}

    board_final = {}

    inherit_product = [] # The stack of inherit-product calls

    product_includes = {} # Other files included by each of the properly imported files

    board_includes = [] # Files included by boardconfig

    with open(filename) as f:

        phase = ""

        for line in csv.reader(f):

            if line[0] == "phase":

                phase = line[1]

            elif line[0] == "val":

                # TOOD: We should skip these somewhere else.

                if line[3].startswith("_ALL_RELEASE_FLAGS"):

                    continue

                if line[3].startswith("PRODUCTS."):

                    continue

                if phase == "PRODUCTS":

                    if line[2] == "initial":

                        product_initial[line[3]] = Variable(line[3], line[4], line[5])

                if phase == "PRODUCT-EXPAND":

                    if line[2] == "final":

                        product_final[line[3]] = Variable(line[3], line[4], line[5])

                if phase == "BOARD":

                    if line[2] == "initial":

                        board_initial[line[3]] = Variable(line[3], line[4], line[5])

                    if line[2] == "final":

                        board_final[line[3]] = Variable(line[3], line[4], line[5])

            elif line[0] == "imported":

                imports = [s.strip() for s in line[1].split()]

                if imports:

                    inherit_product.append(imports)

                    inc = [s.strip() for s in line[2].split()]

                    for f in inc:

                        product_includes.setdefault(imports[0], []).append(f)

            elif line[0] == "board_config_files":

                board_includes += [s.strip() for s in line[1].split()]

    return ProductResult(

        product = lunch[0],

        release = lunch[1],

        variant = lunch[2],

        product_vars = diff(product_initial, product_final),

        board_vars = diff(board_initial, board_final),

        product_graph = NormalizeInheritGraph(inherit_product),

        product_includes = product_includes,

        board_includes = board_includes

    )

async def DumpOneProductConfig(lunch, out_dir) -> ProductResult | ProductError:

    """Print a single config's lunch info to stdout."""

    product, release, variant = lunch

    dumpconfig_file = os.path.join(out_dir, f"{product}-{release}-{variant}.csv")

    # Run get_build_var to bootstrap soong_ui for this target

    env = dict(os.environ)

    env["TARGET_PRODUCT"] = product

    env["TARGET_RELEASE"] = release

    env["TARGET_BUILD_VARIANT"] = variant

    env["OUT_DIR"] = out_dir

    process = await asyncio.create_subprocess_exec(

        "build/soong/bin/get_build_var",

        "TARGET_PRODUCT",

        stdout=subprocess.PIPE,

        stderr=subprocess.STDOUT,

        env=env

    )

    stdout, _ = await process.communicate()

    stdout = stdout.decode()

    if process.returncode != 0:

        return ProductError(

            product = product,

            release = release,

            variant = variant,

            error = stdout

        )

    else:

        # Run kati to extract the data

        process = await asyncio.create_subprocess_exec(

            "prebuilts/build-tools/linux-x86/bin/ckati",

            "-f",

            "build/make/core/dumpconfig.mk",

            f"TARGET_PRODUCT={product}",

            f"TARGET_RELEASE={release}",

            f"TARGET_BUILD_VARIANT={variant}",

            f"DUMPCONFIG_FILE={dumpconfig_file}",

            stdout=subprocess.PIPE,

            stderr=subprocess.STDOUT,

            env=env

        )

        stdout, _ = await process.communicate()

        if process.returncode != 0:

            stdout = stdout.decode()

            return ProductError(

                product = product,

                release = release,

                variant = variant,

                error = stdout

            )

        else:

            # Parse and record the output

            return ParseDump(lunch, dumpconfig_file)

def GetArgs():

    """Parse command line arguments."""

    parser = argparse.ArgumentParser(

            description="Collect all of the make variables from product config.",

            epilog="NOTE: This script must be run from the root of the source tree.")

    parser.add_argument("--lunch", nargs="*")

    parser.add_argument("--dist", action="store_true")

    return parser.parse_args()

async def main():

    args = GetArgs()

    out_dir = buildbot.OutDir()

    if args.dist:

        cm = open(os.path.join(buildbot.DistDir(), "all_product_config.json"), "w")

    else:

        cm = contextlib.nullcontext(sys.stdout)

    with cm as out:

        if args.lunch:

            lunches = [lunch.split("-") for lunch in args.lunch]

            fail = False

            for i in range(len(lunches)):

                if len(lunches[i]) != 3:

                    sys.stderr.write(f"Malformed lunch targets: {args.lunch[i]}\n")

                    fail = True

            if fail:

                sys.exit(1)

            if len(lunches) == 1:

                result = await DumpOneProductConfig(lunches[0], out_dir)

                result.dumpToFile(out)

                out.write("\n")

            else:

                await DumpProductConfigs(out, lunches, out_dir)

        else:

            # All configs mode. This will exec single config mode in parallel

            # for each lunch combo. Write output to $DIST_DIR.

            await DumpProductConfigs(out, GenerateAllLunchTargets(), out_dir)

if __name__ == "__main__":

    asyncio.run(main())

# vim: set syntax=python ts=4 sw=4 sts=4: