Fix python_binary_host module in mixed build. (664a4fd0) · Commits · e / os / android_build_soong

android/bazel.go

+2 −6

Original line number	Diff line number	Diff line
		@@ -309,10 +309,6 @@ var (
		// still be generated via bp2build.
		mixedBuildsDisabledList = []string{
		"libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy
		"func_to_syscall_nrs", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
		"libseccomp_policy_app_zygote_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
		"libseccomp_policy_app_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
		"libseccomp_policy_system_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
		"minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
		}

bazel/aquery.go

+121 −1

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ import (
		"encoding/json"
		"fmt"
		"path/filepath"
		"regexp"
		"strings"

		"github.com/google/blueprint/proptools"
		@@ -59,6 +60,8 @@ type action struct {
		InputDepSetIds []int
		Mnemonic string
		OutputIds []int
		TemplateContent string
		Substitutions []KeyValuePair
		}

		// actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
		@@ -100,6 +103,14 @@ type aqueryArtifactHandler struct {
		artifactIdToPath map[int]string
		}

		// The tokens should be substituted with the value specified here, instead of the
		// one returned in 'substitutions' of TemplateExpand action.
		var TemplateActionOverriddenTokens = map[string]string{
		// Uses "python3" for %python_binary% instead of the value returned by aquery
		// which is "py3wrapper.sh". See removePy3wrapperScript.
		"%python_binary%": "python3",
		}

		func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) {
		pathFragments := map[int]pathFragment{}
		for _, pathFragment := range aqueryResult.PathFragments {
		@@ -163,7 +174,22 @@ func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error)
		}
		}
		}
		return inputPaths, nil

		// Filter out py3wrapper.sh & MANIFEST file. The middleman action returned by aquery
		// for python binary is the input list for a dependent of python binary, since py3wrapper.sh
		// and MANIFEST file could not be created in mixed build, they should be removed from
		// the input paths here.
		py3wrapper := "/py3wrapper.sh"
		manifestFile := regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$")
		filteredInputPaths := []string{}
		for _, path := range inputPaths {
		if strings.HasSuffix(path, py3wrapper) \|\| manifestFile.MatchString(path) {
		continue
		}
		filteredInputPaths = append(filteredInputPaths, path)
		}

		return filteredInputPaths, nil
		}

		func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) {
		@@ -249,6 +275,21 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
		// Use hard links, because some soong actions expect real files (for example, `cp -d`).
		buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -f %[3]s %[2]s", outDir, out, in)
		buildStatement.SymlinkPaths = outputPaths[:]
		} else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 {
		if len(outputPaths) != 1 {
		return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
		}
		expandedTemplateContent := expandTemplateContent(actionEntry)
		command := fmt.Sprintf(`echo "%[1]s" \| sed "s/\\\\n/\\n/g" >> %[2]s && chmod a+x %[2]s`,
		escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
		buildStatement.Command = command
		} else if isPythonZipperAction(actionEntry) {
		if len(inputPaths) < 1 \|\| len(outputPaths) != 1 {
		return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
		}
		buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement)
		buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0])
		addPythonZipFileAsDependencyOfPythonBinary(&buildStatements, outputPaths[0])
		} else if len(actionEntry.Arguments) < 1 {
		return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
		}
		@@ -258,10 +299,89 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
		return buildStatements, nil
		}

		// expandTemplateContent substitutes the tokens in a template.
		func expandTemplateContent(actionEntry action) string {
		replacerString := []string{}
		for _, pair := range actionEntry.Substitutions {
		value := pair.Value
		if val, ok := TemplateActionOverriddenTokens[pair.Key]; ok {
		value = val
		}
		replacerString = append(replacerString, pair.Key, value)
		}
		replacer := strings.NewReplacer(replacerString...)
		return replacer.Replace(actionEntry.TemplateContent)
		}

		func escapeCommandlineArgument(str string) string {
		// \->\\, $->\$, `->\`, "->\", \n->\\n
		replacer := strings.NewReplacer(
		`\`, `\\`,
		`$`, `\$`,
		"`", "\\`",
		`"`, `\"`,
		"\n", "\\n",
		)
		return replacer.Replace(str)
		}

		// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of
		// creating python zip file in mixed build. py3wrapper.sh is returned as input by aquery but
		// there is no action returned by aquery for creating it. So in mixed build "python3" is used
		// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be
		// removed from input paths and command of creating python zip file.
		func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) {
		// Remove from inputs
		py3wrapper := "/py3wrapper.sh"
		filteredInputPaths := []string{}
		for _, path := range bs.InputPaths {
		if !strings.HasSuffix(path, py3wrapper) {
		filteredInputPaths = append(filteredInputPaths, path)
		}
		}
		newInputPaths = filteredInputPaths

		// Remove from command line
		var re = regexp.MustCompile(`\S*` + py3wrapper)
		newCommand = re.ReplaceAllString(bs.Command, "")
		return
		}

		// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory
		// which currently could not be created with aquery output.
		func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string {
		// Unzip the zip file, zipFilePath looks like <python_binary>.zip
		runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles"
		command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName)
		// Create a symblic link in <python_binary>.runfile/, which is the expected structure
		// when running the python binary stub script.
		command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName)
		return bs.Command + " && " + command
		}

		// addPythonZipFileAsDependencyOfPythonBinary adds the action of generating python zip file as dependency of
		// the corresponding action of creating python binary stub script. In mixed build the dependent of python binary depends on
		// the action of createing python binary stub script only, which is not sufficient without the python zip file created.
		func addPythonZipFileAsDependencyOfPythonBinary(buildStatements *[]BuildStatement, pythonZipFilePath string) {
		for i, _ := range *buildStatements {
		if len((buildStatements)[i].OutputPaths) >= 1 && (buildStatements)[i].OutputPaths[0]+".zip" == pythonZipFilePath {
		(buildStatements)[i].InputPaths = append((buildStatements)[i].InputPaths, pythonZipFilePath)
		}
		}
		}

		func isSymlinkAction(a action) bool {
		return a.Mnemonic == "Symlink" \|\| a.Mnemonic == "SolibSymlink"
		}

		func isTemplateExpandAction(a action) bool {
		return a.Mnemonic == "TemplateExpand"
		}

		func isPythonZipperAction(a action) bool {
		return a.Mnemonic == "PythonZipper"
		}

		func shouldSkipAction(a action) bool {
		// TODO(b/180945121): Handle complex symlink actions.
		if a.Mnemonic == "SymlinkTree" \|\| a.Mnemonic == "SourceSymlinkManifest" {

bazel/aquery_test.go

+320 −0

Original line number	Diff line number	Diff line
		@@ -1015,6 +1015,326 @@ func TestSymlinkMultipleOutputs(t *testing.T) {
		assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`)
		}

		func TestTemplateExpandActionSubstitutions(t *testing.T) {
		const inputString = `
		{
		"artifacts": [{
		"id": 1,
		"pathFragmentId": 1
		}],
		"actions": [{
		"targetId": 1,
		"actionKey": "x",
		"mnemonic": "TemplateExpand",
		"configurationId": 1,
		"outputIds": [1],
		"primaryOutputId": 1,
		"executionPlatform": "//build/bazel/platforms:linux_x86_64",
		"templateContent": "Test template substitutions: %token1%, %python_binary%",
		"substitutions": [{
		"key": "%token1%",
		"value": "abcd"
		},{
		"key": "%python_binary%",
		"value": "python3"
		}]
		}],
		"pathFragments": [{
		"id": 1,
		"label": "template_file"
		}]
		}`

		actual, err := AqueryBuildStatements([]byte(inputString))

		if err != nil {
		t.Errorf("Unexpected error %q", err)
		}

		expectedBuildStatements := []BuildStatement{
		BuildStatement{
		Command: "echo \"Test template substitutions: abcd, python3\" \| sed \"s/\\\\\\\\n/\\\\n/g\" >> template_file && " +
		"chmod a+x template_file",
		OutputPaths: []string{"template_file"},
		Mnemonic: "TemplateExpand",
		},
		}
		assertBuildStatements(t, expectedBuildStatements, actual)
		}

		func TestTemplateExpandActionNoOutput(t *testing.T) {
		const inputString = `
		{
		"artifacts": [{
		"id": 1,
		"pathFragmentId": 1
		}],
		"actions": [{
		"targetId": 1,
		"actionKey": "x",
		"mnemonic": "TemplateExpand",
		"configurationId": 1,
		"primaryOutputId": 1,
		"executionPlatform": "//build/bazel/platforms:linux_x86_64",
		"templateContent": "Test template substitutions: %token1%, %python_binary%",
		"substitutions": [{
		"key": "%token1%",
		"value": "abcd"
		},{
		"key": "%python_binary%",
		"value": "python3"
		}]
		}],
		"pathFragments": [{
		"id": 1,
		"label": "template_file"
		}]
		}`

		_, err := AqueryBuildStatements([]byte(inputString))
		assertError(t, err, `Expect 1 output to template expand action, got: output []`)
		}

		func TestPythonZipperActionSuccess(t *testing.T) {
		const inputString = `
		{
		"artifacts": [{
		"id": 1,
		"pathFragmentId": 1
		},{
		"id": 2,
		"pathFragmentId": 2
		},{
		"id": 3,
		"pathFragmentId": 3
		},{
		"id": 4,
		"pathFragmentId": 4
		},{
		"id": 5,
		"pathFragmentId": 10
		}],
		"actions": [{
		"targetId": 1,
		"actionKey": "x",
		"mnemonic": "PythonZipper",
		"configurationId": 1,
		"arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
		"outputIds": [2],
		"inputDepSetIds": [1],
		"primaryOutputId": 2
		}],
		"depSetOfFiles": [{
		"id": 1,
		"directArtifactIds": [4, 3, 5]
		}],
		"pathFragments": [{
		"id": 1,
		"label": "python_binary"
		},{
		"id": 2,
		"label": "python_binary.zip"
		},{
		"id": 3,
		"label": "python_binary.py"
		},{
		"id": 9,
		"label": ".."
		}, {
		"id": 8,
		"label": "bazel_tools",
		"parentId": 9
		}, {
		"id": 7,
		"label": "tools",
		"parentId": 8
		}, {
		"id": 6,
		"label": "zip",
		"parentId": 7
		}, {
		"id": 5,
		"label": "zipper",
		"parentId": 6
		}, {
		"id": 4,
		"label": "zipper",
		"parentId": 5
		},{
		"id": 16,
		"label": "bazel-out"
		},{
		"id": 15,
		"label": "bazel_tools",
		"parentId": 16
		}, {
		"id": 14,
		"label": "k8-fastbuild",
		"parentId": 15
		}, {
		"id": 13,
		"label": "bin",
		"parentId": 14
		}, {
		"id": 12,
		"label": "tools",
		"parentId": 13
		}, {
		"id": 11,
		"label": "python",
		"parentId": 12
		}, {
		"id": 10,
		"label": "py3wrapper.sh",
		"parentId": 11
		}]
		}`
		actual, err := AqueryBuildStatements([]byte(inputString))

		if err != nil {
		t.Errorf("Unexpected error %q", err)
		}

		expectedBuildStatements := []BuildStatement{
		BuildStatement{
		Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " +
		"__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " +
		"../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles",
		InputPaths: []string{"../bazel_tools/tools/zip/zipper/zipper", "python_binary.py"},
		OutputPaths: []string{"python_binary.zip"},
		Mnemonic: "PythonZipper",
		},
		}
		assertBuildStatements(t, expectedBuildStatements, actual)
		}

		func TestPythonZipperActionNoInput(t *testing.T) {
		const inputString = `
		{
		"artifacts": [{
		"id": 1,
		"pathFragmentId": 1
		},{
		"id": 2,
		"pathFragmentId": 2
		}],
		"actions": [{
		"targetId": 1,
		"actionKey": "x",
		"mnemonic": "PythonZipper",
		"configurationId": 1,
		"arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
		"outputIds": [2],
		"primaryOutputId": 2
		}],
		"pathFragments": [{
		"id": 1,
		"label": "python_binary"
		},{
		"id": 2,
		"label": "python_binary.zip"
		}]
		}`
		_, err := AqueryBuildStatements([]byte(inputString))
		assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`)
		}

		func TestPythonZipperActionNoOutput(t *testing.T) {
		const inputString = `
		{
		"artifacts": [{
		"id": 1,
		"pathFragmentId": 1
		},{
		"id": 2,
		"pathFragmentId": 2
		},{
		"id": 3,
		"pathFragmentId": 3
		},{
		"id": 4,
		"pathFragmentId": 4
		},{
		"id": 5,
		"pathFragmentId": 10
		}],
		"actions": [{
		"targetId": 1,
		"actionKey": "x",
		"mnemonic": "PythonZipper",
		"configurationId": 1,
		"arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
		"inputDepSetIds": [1]
		}],
		"depSetOfFiles": [{
		"id": 1,
		"directArtifactIds": [4, 3, 5]
		}],
		"pathFragments": [{
		"id": 1,
		"label": "python_binary"
		},{
		"id": 2,
		"label": "python_binary.zip"
		},{
		"id": 3,
		"label": "python_binary.py"
		},{
		"id": 9,
		"label": ".."
		}, {
		"id": 8,
		"label": "bazel_tools",
		"parentId": 9
		}, {
		"id": 7,
		"label": "tools",
		"parentId": 8
		}, {
		"id": 6,
		"label": "zip",
		"parentId": 7
		}, {
		"id": 5,
		"label": "zipper",
		"parentId": 6
		}, {
		"id": 4,
		"label": "zipper",
		"parentId": 5
		},{
		"id": 16,
		"label": "bazel-out"
		},{
		"id": 15,
		"label": "bazel_tools",
		"parentId": 16
		}, {
		"id": 14,
		"label": "k8-fastbuild",
		"parentId": 15
		}, {
		"id": 13,
		"label": "bin",
		"parentId": 14
		}, {
		"id": 12,
		"label": "tools",
		"parentId": 13
		}, {
		"id": 11,
		"label": "python",
		"parentId": 12
		}, {
		"id": 10,
		"label": "py3wrapper.sh",
		"parentId": 11
		}]
		}`
		_, err := AqueryBuildStatements([]byte(inputString))
		assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`)
		}

		func assertError(t *testing.T, err error, expected string) {
		t.Helper()
		if err == nil {

python/binary.go

+1 −1

Original line number	Diff line number	Diff line
		@@ -143,7 +143,7 @@ var (

		func NewBinary(hod android.HostOrDeviceSupported) (Module, binaryDecorator) {
		module := newModule(hod, android.MultilibFirst)
		decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")}
		decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "", module)}

		module.bootstrapper = decorator
		module.installer = decorator

python/installer.go

+13 −4

Original line number	Diff line number	Diff line
		@@ -36,12 +36,14 @@ type pythonInstaller struct {
		path android.InstallPath

		androidMkSharedLibs []string
		module *Module
		}

		func NewPythonInstaller(dir, dir64 string) *pythonInstaller {
		func NewPythonInstaller(dir, dir64 string, module Module) pythonInstaller {
		return &pythonInstaller{
		dir: dir,
		dir64: dir64,
		module: module,
		}
		}

		@@ -59,8 +61,15 @@ func (installer *pythonInstaller) installDir(ctx android.ModuleContext) android.
		}

		func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) {
		if ctx.ModuleType() == "python_binary_host" && installer.module.MixedBuildsEnabled(ctx) {
		label := installer.module.BazelModuleBase.GetBazelLabel(ctx, installer.module)
		binary, _ := ctx.Config().BazelContext.GetPythonBinary(label, android.GetConfigKey(ctx))
		bazelBinaryOutPath := android.PathForBazelOut(ctx, binary)
		installer.path = ctx.InstallFile(installer.installDir(ctx), bazelBinaryOutPath.Base(), bazelBinaryOutPath)
		} else {
		installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
		}
		}

		func (installer *pythonInstaller) setAndroidMkSharedLibs(sharedLibs []string) {
		installer.androidMkSharedLibs = sharedLibs