Allow choosing which soong_build invocations to run in the debugger (4cbe83ad) · Commits · e / os / android_build_soong

README.md

+36 −11

Original line number	Diff line number	Diff line
		@@ -609,15 +609,15 @@ To load the code of Soong in IntelliJ:
		Content Root, then add the `build/blueprint` directory.
		* Optional: also add the `external/golang-protobuf` directory. In practice,
		IntelliJ seems to work well enough without this, too.

		### Running Soong in a debugger

		To make `soong_build` wait for a debugger connection, install `dlv` and then
		start the build with `SOONG_DELVE=<listen addr>` in the environment.
		For example:
		```bash
		SOONG_DELVE=5006 m nothing
		```
		Both the Android build driver (`soong_ui`) and Soong proper (`soong_build`) are
		Go applications and can be debugged with the help of the standard Go debugger
		called Delve. A client (e.g., IntelliJ IDEA) communicates with Delve via IP port
		that Delve listens to (the port number is passed to it on invocation).

		#### Debugging Android Build Driver ####
		To make `soong_ui` wait for a debugger connection, use the `SOONG_UI_DELVE`
		variable:

		@@ -625,11 +625,28 @@ variable:
		SOONG_UI_DELVE=5006 m nothing
		```

		#### Debugging Soong Proper ####

		setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
		To make `soong_build` wait for a debugger connection, install `dlv` and then
		start the build with `SOONG_DELVE=<listen addr>` in the environment.
		For example:
		```bash
		SOONG_DELVE=5006 m nothing
		```
		Android build driver invokes `soong_build` multiple times, and by default each
		invocation is run in the debugger. Setting `SOONG_DELVE_STEPS` controls which
		invocations are run in the debugger, e.g., running
		```bash
		SOONG_DELVE=2345 SOONG_DELVE_STEPS='build,modulegraph' m
		```
		results in only `build` (main build step) and `modulegraph` being run in the debugger.
		The allowed step names are `api_bp2build`, `bp2build_files`, `bp2build_workspace`,
		`build`, `modulegraph`, `queryview`, `soong_docs`.

		Note setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
		is because in order to debug the binary, it needs to be built with debug
		symbols.

		#### Delve Troubleshooting ####
		To test the debugger connection, run this command:

		```
		@@ -648,15 +665,23 @@ using:
		sudo sysctl -w kernel.yama.ptrace_scope=0
		```

		#### IntelliJ Setup ####
		To connect to the process using IntelliJ:

		* Run -> Edit Configurations...
		* Choose "Go Remote" on the left
		* Click on the "+" buttion on the top-left
		* Give it a nice name and set "Host" to localhost and "Port" to the port in the
		environment variable
		* Give it a nice _name_ and set "Host" to `localhost` and "Port" to the port in the
		environment variable (`SOONG_UI_DELVE` for `soong_ui`, `SOONG_DELVE` for
		`soong_build`)
		* Set the breakpoints where you want application to stop
		* Run the build from the command line
		* In IntelliJ, click Run -> Debug _name_
		* Observe _Connecting..._ message in the debugger pane. It changes to
		_Connected_ once the communication with the debugger has been established; the
		terminal window where the build started will display
		`API server listening at ...` message

		Debugging works far worse than debugging Java, but is sometimes useful.

		Sometimes the `dlv` process hangs on connection. A symptom of this is `dlv`
		spinning a core or two. In that case, `kill -9` `dlv` and try again.

ui/build/soong.go

+138 −115

Original line number	Diff line number	Diff line
		@@ -18,7 +18,6 @@ import (
		"errors"
		"fmt"
		"io/fs"
		"io/ioutil"
		"os"
		"path/filepath"
		"strconv"
		@@ -56,13 +55,13 @@ const (
		bootstrapEpoch = 1
		)

		func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
		func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
		data, err := shared.EnvFileContents(envDeps)
		if err != nil {
		return err
		}

		return ioutil.WriteFile(envFile, data, 0644)
		return os.WriteFile(envFile, data, 0644)
		}

		// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
		@@ -141,7 +140,7 @@ func writeEmptyFile(ctx Context, path string) {
		if exists, err := fileExists(path); err != nil {
		ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
		} else if !exists {
		err = ioutil.WriteFile(path, nil, 0666)
		err = os.WriteFile(path, nil, 0666)
		if err != nil {
		ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
		}
		@@ -157,24 +156,28 @@ func fileExists(path string) (bool, error) {
		return true, nil
		}

		func primaryBuilderInvocation(
		config Config,
		name string,
		output string,
		specificArgs []string,
		description string) bootstrap.PrimaryBuilderInvocation {
		type PrimaryBuilderFactory struct {
		name string
		description string
		config Config
		output string
		specificArgs []string
		debugPort string
		}

		func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
		commonArgs := make([]string, 0, 0)

		if !config.skipSoongTests {
		if !pb.config.skipSoongTests {
		commonArgs = append(commonArgs, "-t")
		}

		commonArgs = append(commonArgs, "-l", filepath.Join(config.FileListDir(), "Android.bp.list"))
		commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
		invocationEnv := make(map[string]string)
		if os.Getenv("SOONG_DELVE") != "" {
		if pb.debugPort != "" {
		//debug mode
		commonArgs = append(commonArgs, "--delve_listen", os.Getenv("SOONG_DELVE"))
		commonArgs = append(commonArgs, "--delve_path", shared.ResolveDelveBinary())
		commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
		"--delve_path", shared.ResolveDelveBinary())
		// GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
		// is useful because the preemption happens by sending SIGURG to the OS
		// thread hosting the goroutine in question and each signal results in
		@@ -188,26 +191,26 @@ func primaryBuilderInvocation(
		}

		var allArgs []string
		allArgs = append(allArgs, specificArgs...)
		allArgs = append(allArgs, pb.specificArgs...)
		allArgs = append(allArgs,
		"--globListDir", name,
		"--globFile", config.NamedGlobFile(name))
		"--globListDir", pb.name,
		"--globFile", pb.config.NamedGlobFile(pb.name))

		allArgs = append(allArgs, commonArgs...)
		allArgs = append(allArgs, environmentArgs(config, name)...)
		allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
		if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
		allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+name)
		allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
		}
		if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
		allArgs = append(allArgs, "--memprofile", profileMem+"."+name)
		allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
		}
		allArgs = append(allArgs, "Android.bp")

		return bootstrap.PrimaryBuilderInvocation{
		Inputs: []string{"Android.bp"},
		Outputs: []string{output},
		Outputs: []string{pb.output},
		Args: allArgs,
		Description: description,
		Description: pb.description,
		// NB: Changing the value of this environment variable will not result in a
		// rebuild. The bootstrap Ninja file will change, but apparently Ninja does
		// not consider changing the pool specified in a statement a change that's
		@@ -270,90 +273,117 @@ func bootstrapBlueprint(ctx Context, config Config) {
		if config.bazelStagingMode {
		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
		}
		queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
		// The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
		// The final workspace will be generated in out/soong/api_bp2build
		apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")

		mainSoongBuildInvocation := primaryBuilderInvocation(
		config,
		soongBuildTag,
		config.SoongNinjaFile(),
		mainSoongBuildExtraArgs,
		fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
		)

		if config.BazelBuildEnabled() {
		// Mixed builds call Bazel from soong_build and they therefore need the
		// Bazel workspace to be available. Make that so by adding a dependency on
		// the bp2build marker file to the action that invokes soong_build .
		mainSoongBuildInvocation.OrderOnlyInputs = append(mainSoongBuildInvocation.OrderOnlyInputs,
		config.Bp2BuildWorkspaceMarkerFile())
		}

		bp2buildInvocation := primaryBuilderInvocation(
		config,
		bp2buildFilesTag,
		config.Bp2BuildFilesMarkerFile(),
		[]string{
		"--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
		pbfs := []PrimaryBuilderFactory{
		{
		name: soongBuildTag,
		description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
		config: config,
		output: config.SoongNinjaFile(),
		specificArgs: mainSoongBuildExtraArgs,
		},
		fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
		)

		bp2buildWorkspaceInvocation := primaryBuilderInvocation(
		config,
		bp2buildWorkspaceTag,
		config.Bp2BuildWorkspaceMarkerFile(),
		[]string{
		"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
		{
		name: bp2buildFilesTag,
		description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
		config: config,
		output: config.Bp2BuildFilesMarkerFile(),
		specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()},
		},
		fmt.Sprintf("Creating Bazel symlink forest"),
		)

		bp2buildWorkspaceInvocation.Inputs = append(bp2buildWorkspaceInvocation.Inputs,
		config.Bp2BuildFilesMarkerFile(), filepath.Join(config.FileListDir(), "bazel.list"))

		jsonModuleGraphInvocation := primaryBuilderInvocation(
		config,
		jsonModuleGraphTag,
		config.ModuleGraphFile(),
		[]string{
		{
		name: bp2buildWorkspaceTag,
		description: "Creating Bazel symlink forest",
		config: config,
		output: config.Bp2BuildWorkspaceMarkerFile(),
		specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()},
		},
		{
		name: jsonModuleGraphTag,
		description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
		config: config,
		output: config.ModuleGraphFile(),
		specificArgs: []string{
		"--module_graph_file", config.ModuleGraphFile(),
		"--module_actions_file", config.ModuleActionsFile(),
		},
		fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
		)

		queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
		queryviewInvocation := primaryBuilderInvocation(
		config,
		queryviewTag,
		config.QueryviewMarkerFile(),
		[]string{
		"--bazel_queryview_dir", queryviewDir,
		},
		fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
		)

		// The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
		// The final workspace will be generated in out/soong/api_bp2build
		apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
		apiBp2buildInvocation := primaryBuilderInvocation(
		config,
		apiBp2buildTag,
		config.ApiBp2buildMarkerFile(),
		[]string{
		"--bazel_api_bp2build_dir", apiBp2buildDir,
		{
		name: queryviewTag,
		description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
		config: config,
		output: config.QueryviewMarkerFile(),
		specificArgs: []string{"--bazel_queryview_dir", queryviewDir},
		},
		fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
		)

		soongDocsInvocation := primaryBuilderInvocation(
		config,
		soongDocsTag,
		config.SoongDocsHtml(),
		[]string{
		"--soong_docs", config.SoongDocsHtml(),
		{
		name: apiBp2buildTag,
		description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
		config: config,
		output: config.ApiBp2buildMarkerFile(),
		specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir},
		},
		fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
		)
		{
		name: soongDocsTag,
		description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
		config: config,
		output: config.SoongDocsHtml(),
		specificArgs: []string{"--soong_docs", config.SoongDocsHtml()},
		},
		}

		// Figure out which invocations will be run under the debugger:
		// * SOONG_DELVE if set specifies listening port
		// * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
		debuggedInvocations := make(map[string]bool)
		delvePort := os.Getenv("SOONG_DELVE")
		if delvePort != "" {
		if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
		var validSteps []string
		for _, pbf := range pbfs {
		debuggedInvocations[pbf.name] = false
		validSteps = append(validSteps, pbf.name)

		}
		for _, step := range strings.Split(steps, ",") {
		if _, ok := debuggedInvocations[step]; ok {
		debuggedInvocations[step] = true
		} else {
		ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
		"Valid steps are %v", step, validSteps)
		}
		}
		} else {
		// SOONG_DELVE_STEPS is not set, run all steps in the debugger
		for _, pbf := range pbfs {
		debuggedInvocations[pbf.name] = true
		}
		}
		}

		var invocations []bootstrap.PrimaryBuilderInvocation
		for _, pbf := range pbfs {
		if debuggedInvocations[pbf.name] {
		pbf.debugPort = delvePort
		}
		pbi := pbf.primaryBuilderInvocation()
		// Some invocations require adjustment:
		switch pbf.name {
		case soongBuildTag:
		if config.BazelBuildEnabled() {
		// Mixed builds call Bazel from soong_build and they therefore need the
		// Bazel workspace to be available. Make that so by adding a dependency on
		// the bp2build marker file to the action that invokes soong_build .
		pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
		}
		case bp2buildWorkspaceTag:
		pbi.Inputs = append(pbi.Inputs,
		config.Bp2BuildFilesMarkerFile(),
		filepath.Join(config.FileListDir(), "bazel.list"))
		}
		invocations = append(invocations, pbi)
		}

		// The glob .ninja files are subninja'd. However, they are generated during
		// the build itself so we write an empty file if the file does not exist yet
		@@ -362,11 +392,11 @@ func bootstrapBlueprint(ctx Context, config Config) {
		writeEmptyFile(ctx, globFile)
		}

		var blueprintArgs bootstrap.Args

		blueprintArgs.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
		blueprintArgs.OutFile = shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja")
		blueprintArgs.EmptyNinjaFile = false
		blueprintArgs := bootstrap.Args{
		ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
		OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
		EmptyNinjaFile: false,
		}

		blueprintCtx := blueprint.NewContext()
		blueprintCtx.SetIgnoreUnknownModuleTypes(true)
		@@ -376,16 +406,9 @@ func bootstrapBlueprint(ctx Context, config Config) {
		outDir: config.OutDir(),
		runGoTests: !config.skipSoongTests,
		// If we want to debug soong_build, we need to compile it for debugging
		debugCompilation: os.Getenv("SOONG_DELVE") != "",
		debugCompilation: delvePort != "",
		subninjas: bootstrapGlobFileList(config),
		primaryBuilderInvocations: []bootstrap.PrimaryBuilderInvocation{
		mainSoongBuildInvocation,
		bp2buildInvocation,
		bp2buildWorkspaceInvocation,
		jsonModuleGraphInvocation,
		queryviewInvocation,
		apiBp2buildInvocation,
		soongDocsInvocation},
		primaryBuilderInvocations: invocations,
		}

		// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little