Merge "Surface module properties as Bazel BUILD target attributes in the Bazel overlay" (81a0b66d) · Commits · e / os / android_build_soong

cmd/soong_build/Android.bp

+3 −0

Original line number	Diff line number	Diff line
		@@ -28,5 +28,8 @@ bootstrap_go_binary {
		"writedocs.go",
		"bazel_overlay.go",
		],
		testSrcs: [
		"bazel_overlay_test.go",
		],
		primaryBuilder: true,
		}

cmd/soong_build/bazel_overlay.go

+313 −37

Original line number	Diff line number	Diff line
		@@ -20,14 +20,17 @@ import (
		"io/ioutil"
		"os"
		"path/filepath"
		"reflect"
		"strings"

		"github.com/google/blueprint"
		"github.com/google/blueprint/proptools"
		)

		const (
		soongModuleLoad = `package(default_visibility = ["//visibility:public"])
		load("//:soong_module.bzl", "soong_module")

		`

		// A BUILD file target snippet representing a Soong module
		@@ -36,15 +39,11 @@ load("//:soong_module.bzl", "soong_module")
		module_name = "%s",
		module_type = "%s",
		module_variant = "%s",
		deps = [
		%s
		],
		)
		`
		deps = %s,
		%s)`

		// The soong_module rule implementation in a .bzl file
		soongModuleBzl = `
		SoongModuleInfo = provider(
		soongModuleBzl = `SoongModuleInfo = provider(
		fields = {
		"name": "Name of module",
		"type": "Type of module",
		@@ -52,7 +51,17 @@ SoongModuleInfo = provider(
		},
		)

		def _soong_module_impl(ctx):
		def _merge_dicts(*dicts):
		"""Adds a list of dictionaries into a single dictionary."""

		# If keys are repeated in multiple dictionaries, the latter one "wins".
		result = {}
		for d in dicts:
		result.update(d)

		return result

		def _generic_soong_module_impl(ctx):
		return [
		SoongModuleInfo(
		name = ctx.attr.module_name,
		@@ -61,14 +70,58 @@ def _soong_module_impl(ctx):
		),
		]

		soong_module = rule(
		implementation = _soong_module_impl,
		attrs = {
		_COMMON_ATTRS = {
		"module_name": attr.string(mandatory = True),
		"module_type": attr.string(mandatory = True),
		"module_variant": attr.string(),
		"deps": attr.label_list(providers = [SoongModuleInfo]),
		},
		}


		generic_soong_module = rule(
		implementation = _generic_soong_module_impl,
		attrs = _COMMON_ATTRS,
		)

		# TODO(jingwen): auto generate Soong module shims
		def _soong_filegroup_impl(ctx):
		return [SoongModuleInfo(),]

		soong_filegroup = rule(
		implementation = _soong_filegroup_impl,
		# Matches https://cs.android.com/android/platform/superproject/+/master:build/soong/android/filegroup.go;l=25-40;drc=6a6478d49e78703ba22a432c41d819c8df79ef6c
		attrs = _merge_dicts(_COMMON_ATTRS, {
		"srcs": attr.string_list(doc = "srcs lists files that will be included in this filegroup"),
		"exclude_srcs": attr.string_list(),
		"path": attr.string(doc = "The base path to the files. May be used by other modules to determine which portion of the path to use. For example, when a filegroup is used as data in a cc_test rule, the base path is stripped off the path and the remaining path is used as the installation directory."),
		"export_to_make_var": attr.string(doc = "Create a make variable with the specified name that contains the list of files in the filegroup, relative to the root of the source tree."),
		})
		)

		soong_module_rule_map = {
		"filegroup": soong_filegroup,
		}

		# soong_module is a macro that supports arbitrary kwargs, and uses module_type to
		# expand to the right underlying shim.
		def soong_module(name, module_type, **kwargs):
		soong_module_rule = soong_module_rule_map.get(module_type)

		if soong_module_rule == None:
		# This module type does not have an existing rule to map to, so use the
		# generic_soong_module rule instead.
		generic_soong_module(
		name = name,
		module_type = module_type,
		module_name = kwargs.pop("module_name", ""),
		module_variant = kwargs.pop("module_variant", ""),
		deps = kwargs.pop("deps", []),
		)
		else:
		soong_module_rule(
		name = name,
		module_type = module_type,
		**kwargs,
		)
		`
		)
		@@ -76,6 +129,7 @@ soong_module = rule(
		func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string {
		name := ""
		if c.ModuleSubDir(logicModule) != "" {
		// TODO(b/162720883): Figure out a way to drop the "--" variant suffixes.
		name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
		} else {
		name = c.ModuleName(logicModule)
		@@ -95,6 +149,153 @@ func packagePath(c *blueprint.Context, logicModule blueprint.Module) string {
		return filepath.Dir(c.BlueprintFile(logicModule))
		}

		func escapeString(s string) string {
		s = strings.ReplaceAll(s, "\\", "\\\\")
		return strings.ReplaceAll(s, "\"", "\\\"")
		}

		func makeIndent(indent int) string {
		if indent < 0 {
		panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent))
		}
		return strings.Repeat(" ", indent)
		}

		// prettyPrint a property value into the equivalent Starlark representation
		// recursively.
		func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
		if isZero(propertyValue) {
		// A property value being set or unset actually matters -- Soong does set default
		// values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at
		// https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480
		//
		// In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default
		// value of unset attributes.
		return "", nil
		}

		var ret string
		switch propertyValue.Kind() {
		case reflect.String:
		ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String()))
		case reflect.Bool:
		ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface()))
		case reflect.Int, reflect.Uint, reflect.Int64:
		ret = fmt.Sprintf("%v", propertyValue.Interface())
		case reflect.Ptr:
		return prettyPrint(propertyValue.Elem(), indent)
		case reflect.Slice:
		ret = "[\n"
		for i := 0; i < propertyValue.Len(); i++ {
		indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1)
		if err != nil {
		return "", err
		}

		if indexedValue != "" {
		ret += makeIndent(indent + 1)
		ret += indexedValue
		ret += ",\n"
		}
		}
		ret += makeIndent(indent)
		ret += "]"
		case reflect.Struct:
		ret = "{\n"
		// Sort and print the struct props by the key.
		structProps := extractStructProperties(propertyValue, indent)
		for _, k := range android.SortedStringKeys(structProps) {
		ret += makeIndent(indent + 1)
		ret += "\"" + k + "\": "
		ret += structProps[k]
		ret += ",\n"
		}
		ret += makeIndent(indent)
		ret += "}"
		case reflect.Interface:
		// TODO(b/164227191): implement pretty print for interfaces.
		// Interfaces are used for for arch, multilib and target properties.
		return "", nil
		default:
		return "", fmt.Errorf(
		"unexpected kind for property struct field: %s", propertyValue.Kind())
		}
		return ret, nil
		}

		func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
		if structValue.Kind() != reflect.Struct {
		panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
		}

		ret := map[string]string{}
		structType := structValue.Type()
		for i := 0; i < structValue.NumField(); i++ {
		field := structType.Field(i)
		if field.PkgPath != "" {
		// Skip unexported fields. Some properties are
		// internal to Soong only, and these fields do not have PkgPath.
		continue
		}
		if proptools.HasTag(field, "blueprint", "mutated") {
		continue
		}

		fieldValue := structValue.Field(i)
		if isZero(fieldValue) {
		// Ignore zero-valued fields
		continue
		}

		propertyName := proptools.PropertyNameForField(field.Name)
		prettyPrintedValue, err := prettyPrint(fieldValue, indent+1)
		if err != nil {
		panic(
		fmt.Errorf(
		"Error while parsing property: %q. %s",
		propertyName,
		err))
		}
		if prettyPrintedValue != "" {
		ret[propertyName] = prettyPrintedValue
		}
		}

		return ret
		}

		func isStructPtr(t reflect.Type) bool {
		return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
		}

		// Generically extract module properties and types into a map, keyed by the module property name.
		func extractModuleProperties(aModule android.Module) map[string]string {
		ret := map[string]string{}

		// Iterate over this android.Module's property structs.
		for _, properties := range aModule.GetProperties() {
		propertiesValue := reflect.ValueOf(properties)
		// Check that propertiesValue is a pointer to the Properties struct, like
		// cc.BaseLinkerProperties or java.CompilerProperties.
		//
		// propertiesValue can also be type-asserted to the structs to
		// manipulate internal props, if needed.
		if isStructPtr(propertiesValue.Type()) {
		structValue := propertiesValue.Elem()
		for k, v := range extractStructProperties(structValue, 0) {
		ret[k] = v
		}
		} else {
		panic(fmt.Errorf(
		"properties must be a pointer to a struct, got %T",
		propertiesValue.Interface()))
		}

		}

		return ret
		}

		func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
		blueprintCtx := ctx.Context
		blueprintCtx.VisitAllModules(func(module blueprint.Module) {
		@@ -103,6 +304,59 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
		panic(err)
		}

		buildFile.Write([]byte(generateSoongModuleTarget(blueprintCtx, module) + "\n\n"))
		buildFile.Close()
		})

		if err := writeReadOnlyFile(bazelOverlayDir, "WORKSPACE", ""); err != nil {
		return err
		}

		if err := writeReadOnlyFile(bazelOverlayDir, "BUILD", ""); err != nil {
		return err
		}

		return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl)
		}

		var ignoredProps map[string]bool = map[string]bool{
		"name": true, // redundant, since this is explicitly generated for every target
		"from": true, // reserved keyword
		"in": true, // reserved keyword
		"arch": true, // interface prop type is not supported yet.
		"multilib": true, // interface prop type is not supported yet.
		"target": true, // interface prop type is not supported yet.
		"visibility": true, // Bazel has native visibility semantics. Handle later.
		}

		func shouldGenerateAttribute(prop string) bool {
		return !ignoredProps[prop]
		}

		// props is an unsorted map. This function ensures that
		// the generated attributes are sorted to ensure determinism.
		func propsToAttributes(props map[string]string) string {
		var attributes string
		for _, propName := range android.SortedStringKeys(props) {
		if shouldGenerateAttribute(propName) {
		attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName])
		}
		}
		return attributes
		}

		// Convert a module and its deps and props into a Bazel macro/rule
		// representation in the BUILD file.
		func generateSoongModuleTarget(
		blueprintCtx *blueprint.Context,
		module blueprint.Module) string {

		var props map[string]string
		if aModule, ok := module.(android.Module); ok {
		props = extractModuleProperties(aModule)
		}
		attributes := propsToAttributes(props)

		// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
		// items, if the modules are added using different DependencyTag. Figure
		// out the implications of that.
		@@ -111,31 +365,22 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
		depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true
		})

		var depLabelList string
		depLabelList := "[\n"
		for depLabel, _ := range depLabels {
		depLabelList += "\"" + depLabel + "\",\n "
		depLabelList += " \""
		depLabelList += depLabel
		depLabelList += "\",\n"
		}
		buildFile.Write([]byte(
		fmt.Sprintf(
		depLabelList += " ]"

		return fmt.Sprintf(
		soongModuleTarget,
		targetNameWithVariant(blueprintCtx, module),
		blueprintCtx.ModuleName(module),
		blueprintCtx.ModuleType(module),
		// misleading name, this actually returns the variant.
		blueprintCtx.ModuleSubDir(module),
		depLabelList)))
		buildFile.Close()
		})

		if err := writeReadOnlyFile(bazelOverlayDir, "WORKSPACE", ""); err != nil {
		return err
		}

		if err := writeReadOnlyFile(bazelOverlayDir, "BUILD", ""); err != nil {
		return err
		}

		return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl)
		depLabelList,
		attributes)
		}

		func buildFileForModule(ctx blueprint.Context, module blueprint.Module) (os.File, error) {
		@@ -171,3 +416,34 @@ func writeReadOnlyFile(dir string, baseName string, content string) error {
		// 0444 is read-only
		return ioutil.WriteFile(workspaceFile, []byte(content), 0444)
		}

		func isZero(value reflect.Value) bool {
		switch value.Kind() {
		case reflect.Func, reflect.Map, reflect.Slice:
		return value.IsNil()
		case reflect.Array:
		valueIsZero := true
		for i := 0; i < value.Len(); i++ {
		valueIsZero = valueIsZero && isZero(value.Index(i))
		}
		return valueIsZero
		case reflect.Struct:
		valueIsZero := true
		for i := 0; i < value.NumField(); i++ {
		if value.Field(i).CanSet() {
		valueIsZero = valueIsZero && isZero(value.Field(i))
		}
		}
		return valueIsZero
		case reflect.Ptr:
		if !value.IsNil() {
		return isZero(reflect.Indirect(value))
		} else {
		return true
		}
		default:
		zeroValue := reflect.Zero(value.Type())
		result := value.Interface() == zeroValue.Interface()
		return result
		}
		}

cmd/soong_build/bazel_overlay_test.go

0 → 100644

+255 −0

Original line number	Diff line number	Diff line
		// Copyright 2020 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"android/soong/android"
		"io/ioutil"
		"os"
		"testing"
		)

		var buildDir string

		func setUp() {
		var err error
		buildDir, err = ioutil.TempDir("", "bazel_overlay_test")
		if err != nil {
		panic(err)
		}
		}

		func tearDown() {
		os.RemoveAll(buildDir)
		}

		func TestMain(m *testing.M) {
		run := func() int {
		setUp()
		defer tearDown()

		return m.Run()
		}

		os.Exit(run())
		}

		type customModule struct {
		android.ModuleBase
		}

		func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		// nothing for now.
		}

		func customModuleFactory() android.Module {
		module := &customModule{}
		android.InitAndroidModule(module)
		return module
		}

		func TestGenerateBazelOverlayFromBlueprint(t *testing.T) {
		testCases := []struct {
		bp string
		expectedBazelTarget string
		}{
		{
		bp: `custom {
		name: "foo",
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		ramdisk: true,
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		ramdisk = True,
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		owner: "a_string_with\"quotes\"_and_\\backslashes\\\\",
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		owner = "a_string_with\"quotes\"_and_\\backslashes\\\\",
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		required: ["bar"],
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		required = [
		"bar",
		],
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		target_required: ["qux", "bazqux"],
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		target_required = [
		"qux",
		"bazqux",
		],
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		dist: {
		targets: ["goal_foo"],
		tag: ".foo",
		},
		dists: [
		{
		targets: ["goal_bar"],
		tag: ".bar",
		},
		],
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		dist = {
		"tag": ".foo",
		"targets": [
		"goal_foo",
		],
		},
		dists = [
		{
		"tag": ".bar",
		"targets": [
		"goal_bar",
		],
		},
		],
		)`,
		},
		{
		bp: `custom {
		name: "foo",
		required: ["bar"],
		target_required: ["qux", "bazqux"],
		ramdisk: true,
		owner: "custom_owner",
		dists: [
		{
		tag: ".tag",
		targets: ["my_goal"],
		},
		],
		}
		`,
		expectedBazelTarget: `soong_module(
		name = "foo",
		module_name = "foo",
		module_type = "custom",
		module_variant = "",
		deps = [
		],
		dists = [
		{
		"tag": ".tag",
		"targets": [
		"my_goal",
		],
		},
		],
		owner = "custom_owner",
		ramdisk = True,
		required = [
		"bar",
		],
		target_required = [
		"qux",
		"bazqux",
		],
		)`,
		},
		}

		for _, testCase := range testCases {
		config := android.TestConfig(buildDir, nil, testCase.bp, nil)
		ctx := android.NewTestContext()
		ctx.RegisterModuleType("custom", customModuleFactory)
		ctx.Register(config)

		_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
		android.FailIfErrored(t, errs)
		_, errs = ctx.PrepareBuildActions(config)
		android.FailIfErrored(t, errs)

		module := ctx.ModuleForTests("foo", "").Module().(*customModule)
		blueprintCtx := ctx.Context.Context

		actualBazelTarget := generateSoongModuleTarget(blueprintCtx, module)
		if actualBazelTarget != testCase.expectedBazelTarget {
		t.Errorf(
		"Expected generated Bazel target to be '%s', got '%s'",
		testCase.expectedBazelTarget,
		actualBazelTarget,
		)
		}
		}
		}