scripts: snapshot of kernel build utility scripts

# Make variables (CC, etc...)

AS		= $(CROSS_COMPILE)as

LD		= $(CROSS_COMPILE)ld

CC		= $(CROSS_COMPILE)gcc

REAL_CC		= $(CROSS_COMPILE)gcc

CPP		= $(CC) -E

AR		= $(CROSS_COMPILE)ar

NM		= $(CROSS_COMPILE)nm

PYTHON3		= python3

CHECK		= sparse

# Use the wrapper for the compiler.  This wrapper scans for new

# warnings and causes the build to stop upon encountering them

CC		= $(srctree)/scripts/gcc-wrapper.py $(REAL_CC)

CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \

		  -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)

NOSTDINC_FLAGS  =

#! /usr/bin/env python2

# SPDX-License-Identifier: GPL-2.0

# Copyright (c) 2009-2015, 2017-18, The Linux Foundation. All rights reserved.

# Build the kernel for all targets using the Android build environment.

from collections import namedtuple

import glob

from optparse import OptionParser

import os

import re

import shutil

import subprocess

import sys

import threading

import Queue

version = 'build-all.py, version 1.99'

build_dir = '../all-kernels'

make_command = ["vmlinux", "modules", "dtbs"]

all_options = {}

compile64 = os.environ.get('CROSS_COMPILE64')

clang_bin = os.environ.get('CLANG_BIN')

def error(msg):

    sys.stderr.write("error: %s\n" % msg)

def fail(msg):

    """Fail with a user-printed message"""

    error(msg)

    sys.exit(1)

if not os.environ.get('CROSS_COMPILE'):

    fail("CROSS_COMPILE must be set in the environment")

def check_kernel():

    """Ensure that PWD is a kernel directory"""

    if not os.path.isfile('MAINTAINERS'):

        fail("This doesn't seem to be a kernel dir")

def check_build():

    """Ensure that the build directory is present."""

    if not os.path.isdir(build_dir):

        try:

            os.makedirs(build_dir)

        except OSError as exc:

            if exc.errno == errno.EEXIST:

                pass

            else:

                raise

failed_targets = []

BuildResult = namedtuple('BuildResult', ['status', 'messages'])

class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])):

    def set_width(self, width):

        self.width = width

    def __enter__(self):

        self.log = open(self.log_name, 'w')

    def __exit__(self, type, value, traceback):

        self.log.close()

    def run(self):

        self.status = None

        messages = ["Building: " + self.short_name]

        def printer(line):

            text = "[%-*s] %s" % (self.width, self.short_name, line)

            messages.append(text)

            self.log.write(text)

            self.log.write('\n')

        for step in self.steps:

            st = step.run(printer)

            if st:

                self.status = BuildResult(self.short_name, messages)

                break

        if not self.status:

            self.status = BuildResult(None, messages)

class BuildTracker:

    """Manages all of the steps necessary to perform a build.  The

    build consists of one or more sequences of steps.  The different

    sequences can be processed independently, while the steps within a

    sequence must be done in order."""

    def __init__(self, parallel_builds):

        self.sequence = []

        self.lock = threading.Lock()

        self.parallel_builds = parallel_builds

    def add_sequence(self, log_name, short_name, steps):

        self.sequence.append(BuildSequence(log_name, short_name, steps))

    def longest_name(self):

        longest = 0

        for seq in self.sequence:

            longest = max(longest, len(seq.short_name))

        return longest

    def __repr__(self):

        return "BuildTracker(%s)" % self.sequence

    def run_child(self, seq):

        seq.set_width(self.longest)

        tok = self.build_tokens.get()

        with self.lock:

            print "Building:", seq.short_name

        with seq:

            seq.run()

            self.results.put(seq.status)

        self.build_tokens.put(tok)

    def run(self):

        self.longest = self.longest_name()

        self.results = Queue.Queue()

        children = []

        errors = []

        self.build_tokens = Queue.Queue()

        nthreads = self.parallel_builds

        print "Building with", nthreads, "threads"

        for i in range(nthreads):

            self.build_tokens.put(True)

        for seq in self.sequence:

            child = threading.Thread(target=self.run_child, args=[seq])

            children.append(child)

            child.start()

        for child in children:

            stats = self.results.get()

            if all_options.verbose:

                with self.lock:

                    for line in stats.messages:

                        print line

                    sys.stdout.flush()

            if stats.status:

                errors.append(stats.status)

        for child in children:

            child.join()

        if errors:

            fail("\n  ".join(["Failed targets:"] + errors))

class PrintStep:

    """A step that just prints a message"""

    def __init__(self, message):

        self.message = message

    def run(self, outp):

        outp(self.message)

class MkdirStep:

    """A step that makes a directory"""

    def __init__(self, direc):

        self.direc = direc

    def run(self, outp):

        outp("mkdir %s" % self.direc)

        os.mkdir(self.direc)

class RmtreeStep:

    def __init__(self, direc):

        self.direc = direc

    def run(self, outp):

        outp("rmtree %s" % self.direc)

        shutil.rmtree(self.direc, ignore_errors=True)

class CopyfileStep:

    def __init__(self, src, dest):

        self.src = src

        self.dest = dest

    def run(self, outp):

        outp("cp %s %s" % (self.src, self.dest))

        shutil.copyfile(self.src, self.dest)

class ExecStep:

    def __init__(self, cmd, **kwargs):

        self.cmd = cmd

        self.kwargs = kwargs

    def run(self, outp):

        outp("exec: %s" % (" ".join(self.cmd),))

        with open('/dev/null', 'r') as devnull:

            proc = subprocess.Popen(self.cmd, stdin=devnull,

                    stdout=subprocess.PIPE,

                    stderr=subprocess.STDOUT,

                    **self.kwargs)

            stdout = proc.stdout

            while True:

                line = stdout.readline()

                if not line:

                    break

                line = line.rstrip('\n')

                outp(line)

            result = proc.wait()

            if result != 0:

                return ('error', result)

            else:

                return None

class Builder():

    def __init__(self, name, defconfig):

        self.name = name

        self.defconfig = defconfig

        self.confname = re.sub('arch/arm[64]*/configs/', '', self.defconfig)

        # Determine if this is a 64-bit target based on the location

        # of the defconfig.

        self.make_env = os.environ.copy()

        if "/arm64/" in defconfig:

            if compile64:

                self.make_env['CROSS_COMPILE'] = compile64

            else:

                fail("Attempting to build 64-bit, without setting CROSS_COMPILE64")

            self.make_env['ARCH'] = 'arm64'

        else:

            self.make_env['ARCH'] = 'arm'

        self.make_env['KCONFIG_NOTIMESTAMP'] = 'true'

        self.log_name = "%s/log-%s.log" % (build_dir, self.name)

    def build(self):

        steps = []

        dest_dir = os.path.join(build_dir, self.name)

        log_name = "%s/log-%s.log" % (build_dir, self.name)

        steps.append(PrintStep('Building %s in %s log %s' %

            (self.name, dest_dir, log_name)))

        if not os.path.isdir(dest_dir):

            steps.append(MkdirStep(dest_dir))

        defconfig = self.defconfig

        dotconfig = '%s/.config' % dest_dir

        savedefconfig = '%s/defconfig' % dest_dir

        staging_dir = 'install_staging'

        modi_dir = '%s' % staging_dir

        hdri_dir = '%s/usr' % staging_dir

        steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir)))

        steps.append(ExecStep(['make', 'O=%s' % dest_dir,

            self.confname], env=self.make_env))

        # Build targets can be dependent upon the completion of

        # previous build targets, so build them one at a time.

        cmd_line = ['make',

            'INSTALL_HDR_PATH=%s' % hdri_dir,

            'INSTALL_MOD_PATH=%s' % modi_dir,

            'O=%s' % dest_dir,

            'REAL_CC=%s' % clang_bin]

        build_targets = []

        for c in make_command:

            if re.match(r'^-{1,2}\w', c):

                cmd_line.append(c)

            else:

                build_targets.append(c)

        for t in build_targets:

            steps.append(ExecStep(cmd_line + [t], env=self.make_env))

        return steps

def scan_configs():

    """Get the full list of defconfigs appropriate for this tree."""

    names = []

    for defconfig in glob.glob('arch/arm*/configs/vendor/*_defconfig'):

        target = os.path.basename(defconfig)[:-10]

        name = target + "-llvm"

        if 'arch/arm64' in defconfig:

            name = name + "-64"

        names.append(Builder(name, defconfig))

    return names

def build_many(targets):

    print "Building %d target(s)" % len(targets)

    # To try and make up for the link phase being serial, try to do

    # two full builds in parallel.  Don't do too many because lots of

    # parallel builds tends to use up available memory rather quickly.

    parallel = 2

    if all_options.jobs and all_options.jobs > 1:

        j = max(all_options.jobs / parallel, 2)

        make_command.append("-j" + str(j))

    tracker = BuildTracker(parallel)

    for target in targets:

        steps = target.build()

        tracker.add_sequence(target.log_name, target.name, steps)

    tracker.run()

def main():

    global make_command

    check_kernel()

    check_build()

    configs = scan_configs()

    usage = ("""

           %prog [options] all                 -- Build all targets

           %prog [options] target target ...   -- List specific targets

           """)

    parser = OptionParser(usage=usage, version=version)

    parser.add_option('--list', action='store_true',

            dest='list',

            help='List available targets')

    parser.add_option('-v', '--verbose', action='store_true',

            dest='verbose',

            help='Output to stdout in addition to log file')

    parser.add_option('-j', '--jobs', type='int', dest="jobs",

            help="Number of simultaneous jobs")

    parser.add_option('-l', '--load-average', type='int',

            dest='load_average',

            help="Don't start multiple jobs unless load is below LOAD_AVERAGE")

    parser.add_option('-k', '--keep-going', action='store_true',

            dest='keep_going', default=False,

            help="Keep building other targets if a target fails")

    parser.add_option('-m', '--make-target', action='append',

            help='Build the indicated make target (default: %s)' %

                 ' '.join(make_command))

    (options, args) = parser.parse_args()

    global all_options

    all_options = options

    if options.list:

        print "Available targets:"

        for target in configs:

            print "   %s" % target.name

        sys.exit(0)

    if options.make_target:

        make_command = options.make_target

    if args == ['all']:

        build_many(configs)

    elif len(args) > 0:

        all_configs = {}

        for t in configs:

            all_configs[t.name] = t

        targets = []

        for t in args:

            if t not in all_configs:

                parser.error("Target '%s' not one of %s" % (t, all_configs.keys()))

            targets.append(all_configs[t])

        build_many(targets)

    else:

        parser.error("Must specify a target to build, or 'all'")

if __name__ == "__main__":

    main()

CP15_BARRIER_EMULATION

DEVKMEM

DEVMEM

HID_A4TECH

HID_ACRUX

HID_BELKIN

HID_CHERRY

HID_CHICONY

HID_CYPRESS

HID_DRAGONRISE

HID_EMS_FF

HID_EZKEY

HID_GREENASIA

HID_GYRATION

HID_HOLTEK

HID_KENSINGTON

HID_KEYTOUCH

HID_KYE

HID_LCPOWER

HID_LOGITECH

HID_MONTEREY

HID_NTRIG

HID_ORTEK

HID_PANTHERLORD

HID_PETALYNX

HID_PICOLCD

HID_PRIMAX

HID_PRODIKEYS

HID_ROCCAT

HID_SAITEK

HID_SAMSUNG

HID_SMARTJOYPLUS

HID_SONY

HID_SPEEDLINK

HID_SUNPLUS

HID_THRUSTMASTER

HID_TIVO

HID_TOPSEED

HID_TWINHAN

HID_UCLOGIC

HID_WACOM

HID_WALTOP

HID_WIIMOTE

HID_ZEROPLUS

HID_ZYDACRON

JOYSTICK_XPAD_FF

JOYSTICK_XPAD_LEDS

KSM

MODULES

PSTORE

SETEND_EMULATION

TABLET_USB_ACECAD

TABLET_USB_AIPTEK

TABLET_USB_GTCO

TABLET_USB_HANWANG

TABLET_USB_KBTAB

USB_CONFIGFS

USB_OTG_WAKELOCK

CGROUP_DEBUG

CP15_BARRIER_EMULATION

DEVKMEM

DEVMEM

HID_A4TECH

HID_ACRUX

HID_BELKIN

HID_CHERRY

HID_CHICONY

HID_CYPRESS

HID_DRAGONRISE

HID_EMS_FF

HID_EZKEY

HID_GREENASIA

HID_GYRATION

HID_HOLTEK

HID_KENSINGTON

HID_KEYTOUCH

HID_KYE

HID_LCPOWER

HID_LOGITECH

HID_MONTEREY

HID_NTRIG

HID_ORTEK

HID_PANTHERLORD

HID_PETALYNX

HID_PICOLCD

HID_PRIMAX

HID_PRODIKEYS

HID_ROCCAT

HID_SAITEK

HID_SAMSUNG

HID_SMARTJOYPLUS

HID_SONY

HID_SPEEDLINK

HID_SUNPLUS

HID_THRUSTMASTER

HID_TIVO

HID_TOPSEED

HID_TWINHAN

HID_UCLOGIC

HID_WACOM

HID_WALTOP

HID_WIIMOTE

HID_ZEROPLUS

HID_ZYDACRON

JOYSTICK_XPAD_FF

JOYSTICK_XPAD_LEDS

KSM

MODULES

PM_DEBUG

PSTORE

SETEND_EMULATION

SUSPEND_TIME

TABLET_USB_ACECAD

TABLET_USB_AIPTEK

TABLET_USB_GTCO

TABLET_USB_HANWANG

TABLET_USB_KBTAB

USB_CONFIGFS

USB_OTG_WAKELOCK

#! /usr/bin/env python

# SPDX-License-Identifier: GPL-2.0

# Copyright (c) 2015, 2018 The Linux Foundation. All rights reserved.

"""

Android kernel configuration validator.

The Android kernel reference trees contain some config stubs of

configuration options that are required for Android to function

correctly, and additional ones that are recommended.

This script can help compare these base configs with the ".config"

output of the compiler to determine if the proper configs are defined.

"""

from collections import namedtuple

from optparse import OptionParser

import re

import sys

version = "check-config.py, version 0.0.1"

req_re = re.compile(r'''^CONFIG_(.*)=(.*)$''')

forb_re = re.compile(r'''^# CONFIG_(.*) is not set$''')

comment_re = re.compile(r'''^(#.*|)$''')

Enabled = namedtuple('Enabled', ['name', 'value'])

Disabled = namedtuple('Disabled', ['name'])

def walk_config(name):

    with open(name, 'r') as fd:

        for line in fd:

            line = line.rstrip()

            m = req_re.match(line)

            if m:

                yield Enabled(m.group(1), m.group(2))

                continue

            m = forb_re.match(line)

            if m:

                yield Disabled(m.group(1))

                continue

            m = comment_re.match(line)

            if m:

                continue

            print "WARNING: Unknown .config line: ", line

class Checker():

    def __init__(self):

        self.required = {}

        self.exempted = set()

        self.forbidden = set()

    def add_required(self, fname):

        for ent in walk_config(fname):

            if type(ent) is Enabled:

                self.required[ent.name] = ent.value

            elif type(ent) is Disabled:

                if ent.name in self.required:

                    del self.required[ent.name]

                self.forbidden.add(ent.name)

    def add_exempted(self, fname):

        with open(fname, 'r') as fd:

            for line in fd:

                line = line.rstrip()

                self.exempted.add(line)

    def check(self, path):

        failure = False

        # Don't run this for mdm targets

        if re.search('mdm', path):

            print "Not applicable to mdm targets... bypassing"

        else:

            for ent in walk_config(path):

                # Go to the next iteration if this config is exempt

                if ent.name in self.exempted:

                   continue

                if type(ent) is Enabled:

                    if ent.name in self.forbidden:

                        print "error: Config should not be present: %s" %ent.name

                        failure = True

                    if ent.name in self.required and ent.value != self.required[ent.name]:

                        print "error: Config has wrong value: %s %s expecting: %s" \

                                 %(ent.name, ent.value, self.required[ent.name])

                        failure = True

                elif type(ent) is Disabled:

                    if ent.name in self.required:

                        print "error: Config should be present, but is disabled: %s" %ent.name

                        failure = True

        if failure:

            sys.exit(1)

def main():

    usage = """%prog [options] path/to/.config"""

    parser = OptionParser(usage=usage, version=version)

    parser.add_option('-r', '--required', dest="required",

            action="append")

    parser.add_option('-e', '--exempted', dest="exempted",

            action="append")

    (options, args) = parser.parse_args()

    if len(args) != 1:

        parser.error("Expecting a single path argument to .config")

    elif options.required is None or options.exempted is None:

        parser.error("Expecting a file containing required configurations")

    ch = Checker()

    for r in options.required:

        ch.add_required(r)

    for e in options.exempted:

        ch.add_exempted(e)

    ch.check(args[0])

if __name__ == '__main__':

    main()