Mixed bazel/soong build prototype for genrule (f3c96efe) · Commits · e / os / android_build_soong

android/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@ bootstrap_go_package {
		"apex.go",
		"api_levels.go",
		"arch.go",
		"bazel_handler.go",
		"bazel_overlay.go",
		"config.go",
		"csuite_config.go",

android/bazel_handler.go

0 → 100644

+239 −0

Original line number	Diff line number	Diff line
		// Copyright 2020 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package android

		import (
		"bytes"
		"errors"
		"fmt"
		"os"
		"os/exec"
		"runtime"
		"strings"
		"sync"
		)

		// Map key to describe bazel cquery requests.
		type cqueryKey struct {
		label string
		starlarkExpr string
		}

		type BazelContext interface {
		// The below methods involve queuing cquery requests to be later invoked
		// by bazel. If any of these methods return (_, false), then the request
		// has been queued to be run later.

		// Returns result files built by building the given bazel target label.
		GetAllFiles(label string) ([]string, bool)

		// TODO(cparsons): Other cquery-related methods should be added here.
		// ** End cquery methods

		// Issues commands to Bazel to receive results for all cquery requests
		// queued in the BazelContext.
		InvokeBazel() error

		// Returns true if bazel is enabled for the given configuration.
		BazelEnabled() bool
		}

		// A context object which tracks queued requests that need to be made to Bazel,
		// and their results after the requests have been made.
		type bazelContext struct {
		homeDir string
		bazelPath string
		outputBase string
		workspaceDir string

		requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
		requestMutex sync.Mutex // requests can be written in parallel

		results map[cqueryKey]string // Results of cquery requests after Bazel invocations
		}

		var _ BazelContext = &bazelContext{}

		// A bazel context to use when Bazel is disabled.
		type noopBazelContext struct{}

		var _ BazelContext = noopBazelContext{}

		// A bazel context to use for tests.
		type MockBazelContext struct {
		AllFiles map[string][]string
		}

		func (m MockBazelContext) GetAllFiles(label string) ([]string, bool) {
		result, ok := m.AllFiles[label]
		return result, ok
		}

		func (m MockBazelContext) InvokeBazel() error {
		panic("unimplemented")
		}

		func (m MockBazelContext) BazelEnabled() bool {
		return true
		}

		var _ BazelContext = MockBazelContext{}

		func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
		starlarkExpr := "', '.join([f.path for f in target.files.to_list()])"
		result, ok := bazelCtx.cquery(label, starlarkExpr)
		if ok {
		bazelOutput := strings.TrimSpace(result)
		return strings.Split(bazelOutput, ", "), true
		} else {
		return nil, false
		}
		}

		func (n noopBazelContext) GetAllFiles(label string) ([]string, bool) {
		panic("unimplemented")
		}

		func (n noopBazelContext) InvokeBazel() error {
		panic("unimplemented")
		}

		func (n noopBazelContext) BazelEnabled() bool {
		return false
		}

		func NewBazelContext(c *config) (BazelContext, error) {
		if c.Getenv("USE_BAZEL") != "1" {
		return noopBazelContext{}, nil
		}

		bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)}
		missingEnvVars := []string{}
		if len(c.Getenv("BAZEL_HOME")) > 1 {
		bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
		} else {
		missingEnvVars = append(missingEnvVars, "BAZEL_HOME")
		}
		if len(c.Getenv("BAZEL_PATH")) > 1 {
		bazelCtx.bazelPath = c.Getenv("BAZEL_PATH")
		} else {
		missingEnvVars = append(missingEnvVars, "BAZEL_PATH")
		}
		if len(c.Getenv("BAZEL_OUTPUT_BASE")) > 1 {
		bazelCtx.outputBase = c.Getenv("BAZEL_OUTPUT_BASE")
		} else {
		missingEnvVars = append(missingEnvVars, "BAZEL_OUTPUT_BASE")
		}
		if len(c.Getenv("BAZEL_WORKSPACE")) > 1 {
		bazelCtx.workspaceDir = c.Getenv("BAZEL_WORKSPACE")
		} else {
		missingEnvVars = append(missingEnvVars, "BAZEL_WORKSPACE")
		}
		if len(missingEnvVars) > 0 {
		return nil, errors.New(fmt.Sprintf("missing required env vars to use bazel: %s", missingEnvVars))
		} else {
		return &bazelCtx, nil
		}
		}

		func (context *bazelContext) BazelEnabled() bool {
		return true
		}

		// Adds a cquery request to the Bazel request queue, to be later invoked, or
		// returns the result of the given request if the request was already made.
		// If the given request was already made (and the results are available), then
		// returns (result, true). If the request is queued but no results are available,
		// then returns ("", false).
		func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) {
		key := cqueryKey{label, starlarkExpr}
		if result, ok := context.results[key]; ok {
		return result, true
		} else {
		context.requestMutex.Lock()
		defer context.requestMutex.Unlock()
		context.requests[key] = true
		return "", false
		}
		}

		func pwdPrefix() string {
		// Darwin doesn't have /proc
		if runtime.GOOS != "darwin" {
		return "PWD=/proc/self/cwd"
		}
		return ""
		}

		func (context *bazelContext) issueBazelCommand(command string, labels []string,
		extraFlags ...string) (string, error) {

		cmdFlags := []string{"--output_base=" + context.outputBase, command}
		cmdFlags = append(cmdFlags, labels...)
		cmdFlags = append(cmdFlags, extraFlags...)

		bazelCmd := exec.Command(context.bazelPath, cmdFlags...)
		bazelCmd.Dir = context.workspaceDir
		bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix())

		var stderr bytes.Buffer
		bazelCmd.Stderr = &stderr

		if output, err := bazelCmd.Output(); err != nil {
		return "", fmt.Errorf("bazel command failed. command: [%s], error [%s]", bazelCmd, stderr)
		} else {
		return string(output), nil
		}
		}

		// Issues commands to Bazel to receive results for all cquery requests
		// queued in the BazelContext.
		func (context *bazelContext) InvokeBazel() error {
		context.results = make(map[cqueryKey]string)

		var labels []string
		var cqueryOutput string
		var err error
		for val, _ := range context.requests {
		labels = append(labels, val.label)

		// TODO(cparsons): Combine requests into a batch cquery request.
		// TODO(cparsons): Use --query_file to avoid command line limits.
		cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label},
		"--output=starlark",
		"--starlark:expr="+val.starlarkExpr)

		if err != nil {
		return err
		} else {
		context.results[val] = string(cqueryOutput)
		}
		}

		// Issue a build command.
		// TODO(cparsons): Invoking bazel execution during soong_build should be avoided;
		// bazel actions should either be added to the Ninja file and executed later,
		// or bazel should handle execution.
		// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
		_, err = context.issueBazelCommand("build", labels)

		if err != nil {
		return err
		}

		// Clear requests.
		context.requests = map[cqueryKey]bool{}
		return nil
		}

android/config.go

+22 −0

Original line number	Diff line number	Diff line
		@@ -85,6 +85,8 @@ type config struct {
		// Only available on configs created by TestConfig
		TestProductVariables *productVariables

		BazelContext BazelContext

		PrimaryBuilder string
		ConfigFileName string
		ProductVariablesFileName string
		@@ -248,6 +250,8 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string
		// Set testAllowNonExistentPaths so that test contexts don't need to specify every path
		// passed to PathForSource or PathForModuleSrc.
		testAllowNonExistentPaths: true,

		BazelContext: noopBazelContext{},
		}
		config.deviceConfig = &deviceConfig{
		config: config,
		@@ -324,6 +328,20 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st
		return testConfig
		}

		// Returns a config object which is "reset" for another bootstrap run.
		// Only per-run data is reset. Data which needs to persist across multiple
		// runs in the same program execution is carried over (such as Bazel context
		// or environment deps).
		func ConfigForAdditionalRun(c Config) (Config, error) {
		newConfig, err := NewConfig(c.srcDir, c.buildDir, c.moduleListFile)
		if err != nil {
		return Config{}, err
		}
		newConfig.BazelContext = c.BazelContext
		newConfig.envDeps = c.envDeps
		return newConfig, nil
		}

		// New creates a new Config object. The srcDir argument specifies the path to
		// the root source directory. It also loads the config file, if found.
		func NewConfig(srcDir, buildDir string, moduleListFile string) (Config, error) {
		@@ -425,6 +443,10 @@ func NewConfig(srcDir, buildDir string, moduleListFile string) (Config, error) {
		Bool(config.productVariables.GcovCoverage) \|\|
		Bool(config.productVariables.ClangCoverage))

		config.BazelContext, err = NewBazelContext(config)
		if err != nil {
		return Config{}, err
		}
		return Config{config}, nil
		}

android/makevars.go

+28 −4

Original line number	Diff line number	Diff line
		@@ -128,7 +128,7 @@ var _ PathContext = MakeVarsContext(nil)
		type MakeVarsProvider func(ctx MakeVarsContext)

		func RegisterMakeVarsProvider(pctx PackageContext, provider MakeVarsProvider) {
		makeVarsProviders = append(makeVarsProviders, makeVarsProvider{pctx, provider})
		makeVarsInitProviders = append(makeVarsInitProviders, makeVarsProvider{pctx, provider})
		}

		// SingletonMakeVarsProvider is a Singleton with an extra method to provide extra values to be exported to Make.
		@@ -142,7 +142,8 @@ type SingletonMakeVarsProvider interface {
		// registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to the list of
		// MakeVarsProviders to run.
		func registerSingletonMakeVarsProvider(singleton SingletonMakeVarsProvider) {
		makeVarsProviders = append(makeVarsProviders, makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)})
		singletonMakeVarsProviders = append(singletonMakeVarsProviders,
		makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)})
		}

		// SingletonmakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider.
		@@ -171,7 +172,11 @@ type makeVarsProvider struct {
		call MakeVarsProvider
		}

		var makeVarsProviders []makeVarsProvider
		// Collection of makevars providers that are registered in init() methods.
		var makeVarsInitProviders []makeVarsProvider

		// Collection of singleton makevars providers that are not registered as part of init() methods.
		var singletonMakeVarsProviders []makeVarsProvider

		type makeVarsContext struct {
		SingletonContext
		@@ -219,7 +224,20 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
		var vars []makeVarsVariable
		var dists []dist
		var phonies []phony
		for _, provider := range makeVarsProviders {
		for _, provider := range append(makeVarsInitProviders) {
		mctx := &makeVarsContext{
		SingletonContext: ctx,
		pctx: provider.pctx,
		}

		provider.call(mctx)

		vars = append(vars, mctx.vars...)
		phonies = append(phonies, mctx.phonies...)
		dists = append(dists, mctx.dists...)
		}

		for _, provider := range append(singletonMakeVarsProviders) {
		mctx := &makeVarsContext{
		SingletonContext: ctx,
		pctx: provider.pctx,
		@@ -232,6 +250,12 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) {
		dists = append(dists, mctx.dists...)
		}

		// Clear singleton makevars providers after use. Since these are in-memory
		// singletons, this ensures state is reset if the build tree is processed
		// multiple times.
		// TODO(cparsons): Clean up makeVarsProviders to be part of the context.
		singletonMakeVarsProviders = nil

		ctx.VisitAllModules(func(m Module) {
		if provider, ok := m.(ModuleMakeVarsProvider); ok && m.Enabled() {
		mctx := &makeVarsContext{

bazel/bazelenv.sh

0 → 100755

+74 −0

Original line number	Diff line number	Diff line
		#!/bin/bash

		# Copyright 2020 Google Inc. All rights reserved.
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		# Helper script for setting environment variables required for Bazel/Soong
		# mixed builds prototype. For development use only.
		#
		# Usage:
		# export BAZEL_PATH=[some_bazel_path] && source bazelenv.sh
		#
		# If BAZEL_PATH is not set, `which bazel` will be used
		# to locate the appropriate bazel to use.


		# Function to find top of the source tree (if $TOP isn't set) by walking up the
		# tree.
		function gettop
		{
		local TOPFILE=build/soong/root.bp
		if [ -n "${TOP-}" -a -f "${TOP-}/${TOPFILE}" ] ; then
		# The following circumlocution ensures we remove symlinks from TOP.
		(cd $TOP; PWD= /bin/pwd)
		else
		if [ -f $TOPFILE ] ; then
		# The following circumlocution (repeated below as well) ensures
		# that we record the true directory name and not one that is
		# faked up with symlink names.
		PWD= /bin/pwd
		else
		local HERE=$PWD
		T=
		while [ $ ! \( -f $TOPFILE $ \) -a $ $PWD != "/" $ ]; do
		\cd ..
		T=`PWD= /bin/pwd -P`
		done
		\cd $HERE
		if [ -f "$T/$TOPFILE" ]; then
		echo $T
		fi
		fi
		fi
		}

		BASE_DIR="$(mktemp -d)"

		if [ -z "$BAZEL_PATH" ] ; then
		export BAZEL_PATH="$(which bazel)"
		fi

		export USE_BAZEL=1
		export BAZEL_HOME="$BASE_DIR/bazelhome"
		export BAZEL_OUTPUT_BASE="$BASE_DIR/output"
		export BAZEL_WORKSPACE="$(gettop)"

		echo "USE_BAZEL=${USE_BAZEL}"
		echo "BAZEL_PATH=${BAZEL_PATH}"
		echo "BAZEL_HOME=${BAZEL_HOME}"
		echo "BAZEL_OUTPUT_BASE=${BAZEL_OUTPUT_BASE}"
		echo "BAZEL_WORKSPACE=${BAZEL_WORKSPACE}"

		mkdir -p $BAZEL_HOME
		mkdir -p $BAZEL_OUTPUT_BASE