Merge changes from topic "go_bp2build" into main (06f3c9bb) · Commits · e / os / android_build_soong

android/bazel_paths.go

+7 −7

Original line number	Diff line number	Diff line
		@@ -198,7 +198,7 @@ func BazelLabelForModuleSrcExcludes(ctx BazelConversionPathContext, paths, exclu
		}
		labels := expandSrcsForBazel(ctx, paths, excluded)
		labels.Excludes = excludeLabels.Includes
		labels = transformSubpackagePaths(ctx, labels)
		labels = TransformSubpackagePaths(ctx.Config(), ctx.ModuleDir(), labels)
		return labels
		}

		@@ -237,7 +237,7 @@ func isPackageBoundary(config Config, prefix string, components []string, compon
		// if the "async_safe" directory is actually a package and not just a directory.
		//
		// In particular, paths that extend into packages are transformed into absolute labels beginning with //.
		func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) bazel.Label {
		func transformSubpackagePath(cfg Config, dir string, path bazel.Label) bazel.Label {
		var newPath bazel.Label

		// Don't transform OriginalModuleName
		@@ -281,7 +281,7 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b
		for i := len(pathComponents) - 1; i >= 0; i-- {
		pathComponent := pathComponents[i]
		var sep string
		if !foundPackageBoundary && isPackageBoundary(ctx.Config(), ctx.ModuleDir(), pathComponents, i) {
		if !foundPackageBoundary && isPackageBoundary(cfg, dir, pathComponents, i) {
		sep = ":"
		foundPackageBoundary = true
		} else {
		@@ -295,7 +295,7 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b
		}
		if foundPackageBoundary {
		// Ensure paths end up looking like //bionic/... instead of //./bionic/...
		moduleDir := ctx.ModuleDir()
		moduleDir := dir
		if strings.HasPrefix(moduleDir, ".") {
		moduleDir = moduleDir[1:]
		}
		@@ -313,13 +313,13 @@ func transformSubpackagePath(ctx BazelConversionPathContext, path bazel.Label) b

		// Transform paths to acknowledge package boundaries
		// See transformSubpackagePath() for more information
		func transformSubpackagePaths(ctx BazelConversionPathContext, paths bazel.LabelList) bazel.LabelList {
		func TransformSubpackagePaths(cfg Config, dir string, paths bazel.LabelList) bazel.LabelList {
		var newPaths bazel.LabelList
		for _, include := range paths.Includes {
		newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(ctx, include))
		newPaths.Includes = append(newPaths.Includes, transformSubpackagePath(cfg, dir, include))
		}
		for _, exclude := range paths.Excludes {
		newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(ctx, exclude))
		newPaths.Excludes = append(newPaths.Excludes, transformSubpackagePath(cfg, dir, exclude))
		}
		return newPaths
		}

android/bazel_paths_test.go

+1 −1

Original line number	Diff line number	Diff line
		@@ -175,7 +175,7 @@ func TestTransformSubpackagePath(t *testing.T) {
		"./z/b.c": "z/b.c",
		}
		for in, out := range pairs {
		actual := transformSubpackagePath(ctx, bazel.Label{Label: in}).Label
		actual := transformSubpackagePath(ctx.Config(), ctx.ModuleDir(), bazel.Label{Label: in}).Label
		if actual != out {
		t.Errorf("expected:\n%v\nactual:\n%v", out, actual)
		}

bp2build/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@ bootstrap_go_package {
		"testing.go",
		],
		deps: [
		"blueprint-bootstrap",
		"soong-aidl-library",
		"soong-android",
		"soong-android-allowlists",
		@@ -37,6 +38,7 @@ bootstrap_go_package {
		"soong-ui-metrics",
		],
		testSrcs: [
		"go_conversion_test.go",
		"aar_conversion_test.go",
		"aidl_library_conversion_test.go",
		"android_app_certificate_conversion_test.go",

bp2build/build_conversion.go

+243 −1

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import (
		"android/soong/starlark_fmt"
		"android/soong/ui/metrics/bp2build_metrics_proto"
		"github.com/google/blueprint"
		"github.com/google/blueprint/bootstrap"
		"github.com/google/blueprint/proptools"
		)

		@@ -252,6 +253,235 @@ func (r conversionResults) BuildDirToTargets() map[string]BazelTargets {
		return r.buildFileToTargets
		}

		// struct to store state of go bazel targets
		// this implements bp2buildModule interface and is passed to generateBazelTargets
		type goBazelTarget struct {
		targetName string
		targetPackage string
		bazelRuleClass string
		bazelRuleLoadLocation string
		bazelAttributes []interface{}
		}

		var _ bp2buildModule = (*goBazelTarget)(nil)

		func (g goBazelTarget) TargetName() string {
		return g.targetName
		}

		func (g goBazelTarget) TargetPackage() string {
		return g.targetPackage
		}

		func (g goBazelTarget) BazelRuleClass() string {
		return g.bazelRuleClass
		}

		func (g goBazelTarget) BazelRuleLoadLocation() string {
		return g.bazelRuleLoadLocation
		}

		func (g goBazelTarget) BazelAttributes() []interface{} {
		return g.bazelAttributes
		}

		// Creates a target_compatible_with entry that is not compatible with android
		func targetNotCompatibleWithAndroid() bazel.LabelListAttribute {
		ret := bazel.LabelListAttribute{}
		ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsAndroid,
		bazel.MakeLabelList(
		[]bazel.Label{
		bazel.Label{
		Label: "@platforms//:incompatible",
		},
		},
		),
		)
		return ret
		}

		// helper function to return labels for srcs used in bootstrap_go_package and bootstrap_go_binary
		// this function has the following limitations which make it unsuitable for widespread use
		// - wildcard patterns in srcs
		// This is ok for go since build/blueprint does not support it.
		//
		// Prefer to use `BazelLabelForModuleSrc` instead
		func goSrcLabels(cfg android.Config, moduleDir string, srcs []string, linuxSrcs, darwinSrcs []string) bazel.LabelListAttribute {
		labels := func(srcs []string) bazel.LabelList {
		ret := []bazel.Label{}
		for _, src := range srcs {
		srcLabel := bazel.Label{
		Label: src,
		}
		ret = append(ret, srcLabel)
		}
		// Respect package boundaries
		return android.TransformSubpackagePaths(
		cfg,
		moduleDir,
		bazel.MakeLabelList(ret),
		)
		}

		ret := bazel.LabelListAttribute{}
		// common
		ret.SetSelectValue(bazel.NoConfigAxis, "", labels(srcs))
		// linux
		ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsLinux, labels(linuxSrcs))
		// darwin
		ret.SetSelectValue(bazel.OsConfigurationAxis, bazel.OsDarwin, labels(darwinSrcs))
		return ret
		}

		func goDepLabels(deps []string, goModulesMap nameToGoLibraryModule) bazel.LabelListAttribute {
		labels := []bazel.Label{}
		for _, dep := range deps {
		moduleDir := goModulesMap[dep].Dir
		if moduleDir == "." {
		moduleDir = ""
		}
		label := bazel.Label{
		Label: fmt.Sprintf("//%s:%s", moduleDir, dep),
		}
		labels = append(labels, label)
		}
		return bazel.MakeLabelListAttribute(bazel.MakeLabelList(labels))
		}

		// attributes common to blueprint_go_binary and bootstap_go_package
		type goAttributes struct {
		Importpath bazel.StringAttribute
		Srcs bazel.LabelListAttribute
		Deps bazel.LabelListAttribute
		Target_compatible_with bazel.LabelListAttribute
		}

		func generateBazelTargetsGoPackage(ctx android.Context, g bootstrap.GoPackage, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
		ca := android.CommonAttributes{
		Name: g.Name(),
		}

		// For this bootstrap_go_package dep chain,
		// A --> B --> C ( ---> depends on)
		// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
		// Bazel OTOH
		// 1. requires C to be listed in `deps` expllicity.
		// 2. does not require C to be listed if src of A does not import C
		//
		// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
		transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)

		ga := goAttributes{
		Importpath: bazel.StringAttribute{
		Value: proptools.StringPtr(g.GoPkgPath()),
		},
		Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
		Deps: goDepLabels(
		android.FirstUniqueStrings(transitiveDeps),
		goModulesMap,
		),
		Target_compatible_with: targetNotCompatibleWithAndroid(),
		}

		lib := goBazelTarget{
		targetName: g.Name(),
		targetPackage: ctx.ModuleDir(g),
		bazelRuleClass: "go_library",
		bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
		bazelAttributes: []interface{}{&ca, &ga},
		}
		// TODO - b/284483729: Create go_test target from testSrcs
		libTarget, err := generateBazelTarget(ctx, lib)
		if err != nil {
		return []BazelTarget{}, []error{err}
		}
		return []BazelTarget{libTarget}, nil
		}

		type goLibraryModule struct {
		Dir string
		Deps []string
		}

		type nameToGoLibraryModule map[string]goLibraryModule

		// Visit each module in the graph
		// If a module is of type `bootstrap_go_package`, return a map containing metadata like its dir and deps
		func createGoLibraryModuleMap(ctx *android.Context) nameToGoLibraryModule {
		ret := nameToGoLibraryModule{}
		ctx.VisitAllModules(func(m blueprint.Module) {
		moduleType := ctx.ModuleType(m)
		// We do not need to store information about blueprint_go_binary since it does not have any rdeps
		if moduleType == "bootstrap_go_package" {
		ret[m.Name()] = goLibraryModule{
		Dir: ctx.ModuleDir(m),
		Deps: m.(*bootstrap.GoPackage).Deps(),
		}
		}
		})
		return ret
		}

		// Returns the deps in the transitive closure of a go target
		func transitiveGoDeps(directDeps []string, goModulesMap nameToGoLibraryModule) []string {
		allDeps := directDeps
		i := 0
		for i < len(allDeps) {
		curr := allDeps[i]
		allDeps = append(allDeps, goModulesMap[curr].Deps...)
		i += 1
		}
		allDeps = android.SortedUniqueStrings(allDeps)
		return allDeps
		}

		func generateBazelTargetsGoBinary(ctx android.Context, g bootstrap.GoBinary, goModulesMap nameToGoLibraryModule) ([]BazelTarget, []error) {
		ca := android.CommonAttributes{
		Name: g.Name(),
		}

		// For this bootstrap_go_package dep chain,
		// A --> B --> C ( ---> depends on)
		// Soong provides the convenience of only listing B as deps of A even if a src file of A imports C
		// Bazel OTOH
		// 1. requires C to be listed in `deps` expllicity.
		// 2. does not require C to be listed if src of A does not import C
		//
		// bp2build does not have sufficient info on whether C is a direct dep of A or not, so for now collect all transitive deps and add them to deps
		transitiveDeps := transitiveGoDeps(g.Deps(), goModulesMap)

		ga := goAttributes{
		Srcs: goSrcLabels(ctx.Config(), ctx.ModuleDir(g), g.Srcs(), g.LinuxSrcs(), g.DarwinSrcs()),
		Deps: goDepLabels(transitiveDeps, goModulesMap),
		Target_compatible_with: targetNotCompatibleWithAndroid(),
		}

		bin := goBazelTarget{
		targetName: g.Name(),
		targetPackage: ctx.ModuleDir(g),
		bazelRuleClass: "go_binary",
		bazelRuleLoadLocation: "@io_bazel_rules_go//go:def.bzl",
		bazelAttributes: []interface{}{&ca, &ga},
		}
		// TODO - b/284483729: Create go_test target from testSrcs
		binTarget, err := generateBazelTarget(ctx, bin)
		if err != nil {
		return []BazelTarget{}, []error{err}
		}
		return []BazelTarget{binTarget}, nil
		}

		var (
		// TODO - b/284483729: Remove this denyilst
		// Temporary denylist of go binaries that are currently used in mixed builds
		// This denylist allows us to rollout bp2build converters for go targets without affecting mixed builds
		goBinaryDenylist = []string{
		"soong_zip",
		"zip2zip",
		"bazel_notice_gen",
		}
		)

		func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (conversionResults, []error) {
		buildFileToTargets := make(map[string]BazelTargets)

		@@ -262,6 +492,10 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers

		var errs []error

		// Visit go libraries in a pre-run and store its state in a map
		// The time complexity remains O(N), and this does not add significant wall time.
		nameToGoLibMap := createGoLibraryModuleMap(ctx.Context())

		bpCtx := ctx.Context()
		bpCtx.VisitAllModules(func(m blueprint.Module) {
		dir := bpCtx.ModuleDir(m)
		@@ -269,6 +503,7 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
		dirs[dir] = true

		var targets []BazelTarget
		var targetErrs []error

		switch ctx.Mode() {
		case Bp2Build:
		@@ -317,7 +552,6 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
		return
		}
		}
		var targetErrs []error
		targets, targetErrs = generateBazelTargets(bpCtx, aModule)
		errs = append(errs, targetErrs...)
		for _, t := range targets {
		@@ -336,6 +570,14 @@ func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (convers
		metrics.AddUnconvertedModule(m, moduleType, dir, *reason)
		}
		return
		} else if glib, ok := m.(*bootstrap.GoPackage); ok {
		targets, targetErrs = generateBazelTargetsGoPackage(bpCtx, glib, nameToGoLibMap)
		errs = append(errs, targetErrs...)
		metrics.IncrementRuleClassCount("go_library")
		} else if gbin, ok := m.(*bootstrap.GoBinary); ok && !android.InList(m.Name(), goBinaryDenylist) {
		targets, targetErrs = generateBazelTargetsGoBinary(bpCtx, gbin, nameToGoLibMap)
		errs = append(errs, targetErrs...)
		metrics.IncrementRuleClassCount("go_binary")
		} else {
		metrics.AddUnconvertedModule(m, moduleType, dir, android.UnconvertedReason{
		ReasonType: int(bp2build_metrics_proto.UnconvertedReasonType_TYPE_UNSUPPORTED),

bp2build/go_conversion_test.go

0 → 100644

+150 −0

Original line number	Diff line number	Diff line
		// Copyright 2023 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package bp2build

		import (
		"testing"

		"github.com/google/blueprint/bootstrap"

		"android/soong/android"
		)

		func runGoTests(t *testing.T, tc Bp2buildTestCase) {
		RunBp2BuildTestCase(t, func(ctx android.RegistrationContext) {
		tCtx := ctx.(*android.TestContext)
		bootstrap.RegisterGoModuleTypes(tCtx.Context.Context) // android.TestContext --> android.Context --> blueprint.Context
		}, tc)
		}

		func TestConvertGoPackage(t *testing.T) {
		bp := `
		bootstrap_go_package {
		name: "foo",
		pkgPath: "android/foo",
		deps: [
		"bar",
		],
		srcs: [
		"foo1.go",
		"foo2.go",
		],
		linux: {
		srcs: [
		"foo_linux.go",
		],
		},
		darwin: {
		srcs: [
		"foo_darwin.go",
		],
		},
		testSrcs: [
		"foo1_test.go",
		"foo2_test.go",
		],
		}
		`
		depBp := `
		bootstrap_go_package {
		name: "bar",
		}
		`
		t.Parallel()
		runGoTests(t, Bp2buildTestCase{
		Description: "Convert bootstrap_go_package to go_library",
		ModuleTypeUnderTest: "bootrstap_go_package",
		Blueprint: bp,
		Filesystem: map[string]string{
		"bar/Android.bp": depBp, // Put dep in Android.bp to reduce boilerplate in ExpectedBazelTargets
		},
		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_library", "foo",
		AttrNameToString{
		"deps": `["//bar:bar"]`,
		"importpath": `"android/foo"`,
		"srcs": `[
		"foo1.go",
		"foo2.go",
		] + select({
		"//build/bazel/platforms/os:darwin": ["foo_darwin.go"],
		"//build/bazel/platforms/os:linux_glibc": ["foo_linux.go"],
		"//conditions:default": [],
		})`,
		},
		android.HostSupported,
		)},
		})
		}

		func TestConvertGoBinaryWithTransitiveDeps(t *testing.T) {
		bp := `
		blueprint_go_binary {
		name: "foo",
		srcs: ["main.go"],
		deps: ["bar"],
		}
		`
		depBp := `
		bootstrap_go_package {
		name: "bar",
		deps: ["baz"],
		}
		bootstrap_go_package {
		name: "baz",
		}
		`
		t.Parallel()
		runGoTests(t, Bp2buildTestCase{
		Description: "Convert blueprint_go_binary to go_binary",
		Blueprint: bp,
		Filesystem: map[string]string{
		"bar/Android.bp": depBp, // Put dep in Android.bp to reduce boilerplate in ExpectedBazelTargets
		},
		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_binary", "foo",
		AttrNameToString{
		"deps": `[
		"//bar:bar",
		"//bar:baz",
		]`,
		"srcs": `["main.go"]`,
		},
		android.HostSupported,
		)},
		})
		}

		func TestConvertGoBinaryWithSrcInDifferentPackage(t *testing.T) {
		bp := `
		blueprint_go_binary {
		name: "foo",
		srcs: ["subdir/main.go"],
		}
		`
		t.Parallel()
		runGoTests(t, Bp2buildTestCase{
		Description: "Convert blueprint_go_binary with src in different package",
		Blueprint: bp,
		Filesystem: map[string]string{
		"subdir/Android.bp": "",
		},
		ExpectedBazelTargets: []string{makeBazelTargetHostOrDevice("go_binary", "foo",
		AttrNameToString{
		"deps": `[]`,
		"srcs": `["//subdir:main.go"]`,
		},
		android.HostSupported,
		)},
		})
		}