Add crunch-flags and build-flag binaries (b9014c74) · Commits · e / os / android_build_soong

cmd/release_config/build_flag/Android.bp

0 → 100644

+32 −0

Original line number	Diff line number	Diff line
		package {
		default_applicable_licenses: ["Android-Apache-2.0"],
		}

		blueprint_go_binary {
		name: "build-flag",
		deps: [
		"golang-protobuf-encoding-prototext",
		"golang-protobuf-reflect-protoreflect",
		"golang-protobuf-runtime-protoimpl",
		"soong-cmd-release_config-proto",
		"soong-cmd-release_config-lib",
		],
		srcs: [
		"main.go",
		],
		}

		bootstrap_go_package {
		name: "soong-cmd-release_config-build_flag",
		pkgPath: "android/soong/cmd/release_config/build_flag",
		deps: [
		"golang-protobuf-encoding-prototext",
		"golang-protobuf-reflect-protoreflect",
		"golang-protobuf-runtime-protoimpl",
		"soong-cmd-release_config-proto",
		"soong-cmd-release_config-lib",
		],
		srcs: [
		"main.go",
		],
		}

cmd/release_config/build_flag/main.go

0 → 100644

+229 −0

Original line number	Diff line number	Diff line
		package main

		import (
		"flag"
		"fmt"
		"os"
		"path/filepath"
		"strings"

		rc_lib "android/soong/cmd/release_config/release_config_lib"
		rc_proto "android/soong/cmd/release_config/release_config_proto"

		"google.golang.org/protobuf/proto"
		)

		type Flags struct {
		// The path to the top of the workspace. Default: ".".
		top string

		// Pathlist of release config map textproto files.
		// If not specified, then the value is (if present):
		// - build/release/release_config_map.textproto
		// - vendor/google_shared/build/release/release_config_map.textproto
		// - vendor/google/release/release_config_map.textproto
		//
		// Additionally, any maps specified in the environment variable
		// `PRODUCT_RELEASE_CONFIG_MAPS` are used.
		maps rc_lib.StringList

		// Output directory (relative to `top`).
		outDir string

		// Which $TARGET_RELEASE(s) should we use. Some commands will only
		// accept one value, others also accept `--release --all`.
		targetReleases rc_lib.StringList

		// Disable warning messages
		quiet bool
		}

		type CommandFunc func(*rc_lib.ReleaseConfigs, Flags, string, []string) error

		var commandMap map[string]CommandFunc = map[string]CommandFunc{
		"get": GetCommand,
		"set": SetCommand,
		"trace": GetCommand, // Also handled by GetCommand
		}

		// Find the top of the release config contribution directory.
		// Returns the parent of the flag_declarations and flag_values directories.
		func GetMapDir(path string) (string, error) {
		for p := path; p != "."; p = filepath.Dir(p) {
		switch filepath.Base(p) {
		case "flag_declarations":
		return filepath.Dir(p), nil
		case "flag_values":
		return filepath.Dir(p), nil
		}
		}
		return "", fmt.Errorf("Could not determine directory from %s", path)
		}

		func MarshalFlagValue(config *rc_lib.ReleaseConfig, name string) (ret string, err error) {
		fa, ok := config.FlagArtifacts[name]
		if !ok {
		return "", fmt.Errorf("%s not found in %s", name, config.Name)
		}
		return rc_lib.MarshalValue(fa.Value), nil
		}

		func GetReleaseArgs(configs rc_lib.ReleaseConfigs, commonFlags Flags) ([]rc_lib.ReleaseConfig, error) {
		var all bool
		relFlags := flag.NewFlagSet("set", flag.ExitOnError)
		relFlags.BoolVar(&all, "all", false, "Display all flags")
		relFlags.Parse(commonFlags.targetReleases)
		var ret []*rc_lib.ReleaseConfig
		if all {
		for _, config := range configs.ReleaseConfigs {
		ret = append(ret, config)
		}
		return ret, nil
		}
		for _, arg := range relFlags.Args() {
		config, err := configs.GetReleaseConfig(arg)
		if err != nil {
		return nil, err
		}
		ret = append(ret, config)
		}
		return ret, nil
		}

		func GetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error {
		isTrace := cmd == "trace"
		var all bool
		getFlags := flag.NewFlagSet("set", flag.ExitOnError)
		getFlags.BoolVar(&all, "all", false, "Display all flags")
		getFlags.Parse(args)
		args = getFlags.Args()

		releaseConfigList, err := GetReleaseArgs(configs, commonFlags)
		if err != nil {
		return err
		}
		if isTrace && len(releaseConfigList) > 1 {
		return fmt.Errorf("trace command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " "))
		}

		if all {
		args = []string{}
		for _, fa := range configs.FlagArtifacts {
		args = append(args, *fa.FlagDeclaration.Name)
		}
		}

		showName := len(releaseConfigList) > 1 \|\| len(args) > 1
		for _, config := range releaseConfigList {
		var configName string
		if len(releaseConfigList) > 1 {
		configName = fmt.Sprintf("%s.", config.Name)
		}
		for _, arg := range args {
		val, err := MarshalFlagValue(config, arg)
		if err != nil {
		return err
		}
		if showName {
		fmt.Printf("%s%s=%s\n", configName, arg, val)
		} else {
		fmt.Printf("%s\n", val)
		}
		if isTrace {
		for _, trace := range config.FlagArtifacts[arg].Traces {
		fmt.Printf(" => \"%s\" in %s\n", rc_lib.MarshalValue(trace.Value), *trace.Source)
		}
		}
		}
		}
		return nil
		}

		func SetCommand(configs *rc_lib.ReleaseConfigs, commonFlags Flags, cmd string, args []string) error {
		var valueDir string
		if len(commonFlags.targetReleases) > 1 {
		return fmt.Errorf("set command only allows one --release argument. Got: %s", strings.Join(commonFlags.targetReleases, " "))
		}
		targetRelease := commonFlags.targetReleases[0]

		setFlags := flag.NewFlagSet("set", flag.ExitOnError)
		setFlags.StringVar(&valueDir, "dir", "", "Directory in which to place the value")
		setFlags.Parse(args)
		setArgs := setFlags.Args()
		if len(setArgs) != 2 {
		return fmt.Errorf("set command expected flag and value, got: %s", strings.Join(setArgs, " "))
		}
		name := setArgs[0]
		value := setArgs[1]
		release, err := configs.GetReleaseConfig(targetRelease)
		targetRelease = release.Name
		if err != nil {
		return err
		}
		flagArtifact, ok := release.FlagArtifacts[name]
		if !ok {
		return fmt.Errorf("Unknown build flag %s", name)
		}
		if valueDir == "" {
		mapDir, err := GetMapDir(*flagArtifact.Traces[len(flagArtifact.Traces)-1].Source)
		if err != nil {
		return err
		}
		valueDir = mapDir
		}

		flagValue := &rc_proto.FlagValue{
		Name: proto.String(name),
		Value: rc_lib.UnmarshalValue(value),
		}
		flagPath := filepath.Join(valueDir, "flag_values", targetRelease, fmt.Sprintf("%s.textproto", name))
		return rc_lib.WriteMessage(flagPath, flagValue)
		}

		func main() {
		var err error
		var commonFlags Flags
		var configs *rc_lib.ReleaseConfigs

		outEnv := os.Getenv("OUT_DIR")
		if outEnv == "" {
		outEnv = "out"
		}
		// Handle the common arguments
		flag.StringVar(&commonFlags.top, "top", ".", "path to top of workspace")
		flag.BoolVar(&commonFlags.quiet, "quiet", false, "disable warning messages")
		flag.Var(&commonFlags.maps, "map", "path to a release_config_map.textproto. may be repeated")
		flag.StringVar(&commonFlags.outDir, "out_dir", rc_lib.GetDefaultOutDir(), "basepath for the output. Multiple formats are created")
		flag.Var(&commonFlags.targetReleases, "release", "TARGET_RELEASE for this build")
		flag.Parse()

		if commonFlags.quiet {
		rc_lib.DisableWarnings()
		}

		if len(commonFlags.targetReleases) == 0 {
		commonFlags.targetReleases = rc_lib.StringList{"trunk_staging"}
		}

		if err = os.Chdir(commonFlags.top); err != nil {
		panic(err)
		}

		// Get the current state of flagging.
		relName := commonFlags.targetReleases[0]
		if relName == "--all" \|\| relName == "-all" {
		// If the users said `--release --all`, grab trunk staging for simplicity.
		relName = "trunk_staging"
		}
		configs, err = rc_lib.ReadReleaseConfigMaps(commonFlags.maps, relName)
		if err != nil {
		panic(err)
		}

		if cmd, ok := commandMap[flag.Arg(0)]; ok {
		args := flag.Args()
		if err = cmd(configs, commonFlags, args[0], args[1:]); err != nil {
		panic(err)
		}
		}
		}

cmd/release_config/crunch_flags/Android.bp

0 → 100644

+32 −0

Original line number	Diff line number	Diff line
		package {
		default_applicable_licenses: ["Android-Apache-2.0"],
		}

		blueprint_go_binary {
		name: "crunch-flags",
		deps: [
		"golang-protobuf-encoding-prototext",
		"golang-protobuf-reflect-protoreflect",
		"golang-protobuf-runtime-protoimpl",
		"soong-cmd-release_config-lib",
		"soong-cmd-release_config-proto",
		],
		srcs: [
		"main.go",
		],
		}

		bootstrap_go_package {
		name: "soong-cmd-release_config-crunch_flags",
		pkgPath: "android/soong/cmd/release_config/crunch_flags",
		deps: [
		"golang-protobuf-encoding-prototext",
		"golang-protobuf-reflect-protoreflect",
		"golang-protobuf-runtime-protoimpl",
		"soong-cmd-release_config-lib",
		"soong-cmd-release_config-proto",
		],
		srcs: [
		"main.go",
		],
		}

cmd/release_config/crunch_flags/main.go

0 → 100644

+359 −0

Original line number	Diff line number	Diff line
		package main

		import (
		"flag"
		"fmt"
		"io/fs"
		"os"
		"path/filepath"
		"regexp"
		"strings"

		rc_lib "android/soong/cmd/release_config/release_config_lib"
		rc_proto "android/soong/cmd/release_config/release_config_proto"
		"google.golang.org/protobuf/encoding/prototext"
		"google.golang.org/protobuf/proto"
		)

		// When a flag declaration has an initial value that is a string, the default workflow is PREBUILT.
		// If the flag name starts with any of prefixes in manualFlagNamePrefixes, it is MANUAL.
		var manualFlagNamePrefixes []string = []string{
		"RELEASE_ACONFIG_",
		"RELEASE_PLATFORM_",
		}

		var defaultFlagNamespace string = "android_UNKNOWN"

		func RenameNext(name string) string {
		if name == "next" {
		return "ap3a"
		}
		return name
		}

		func WriteFile(path string, message proto.Message) error {
		data, err := prototext.MarshalOptions{Multiline: true}.Marshal(message)
		if err != nil {
		return err
		}

		err = os.MkdirAll(filepath.Dir(path), 0775)
		if err != nil {
		return err
		}
		return os.WriteFile(path, data, 0644)
		}

		func WalkValueFiles(dir string, Func fs.WalkDirFunc) error {
		valPath := filepath.Join(dir, "build_config")
		if _, err := os.Stat(valPath); err != nil {
		fmt.Printf("%s not found, ignoring.\n", valPath)
		return nil
		}

		return filepath.WalkDir(valPath, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
		return err
		}
		if strings.HasSuffix(d.Name(), ".scl") && d.Type().IsRegular() {
		return Func(path, d, err)
		}
		return nil
		})
		}

		func ProcessBuildFlags(dir string, namespaceMap map[string]string) error {
		var rootAconfigModule string

		path := filepath.Join(dir, "build_flags.scl")
		if _, err := os.Stat(path); err != nil {
		fmt.Printf("%s not found, ignoring.\n", path)
		return nil
		} else {
		fmt.Printf("Processing %s\n", path)
		}
		commentRegexp, err := regexp.Compile("^[[:space:]]*#(?<comment>.+)")
		if err != nil {
		return err
		}
		declRegexp, err := regexp.Compile("^[[:space:]]flag.\"(?<name>[A-Z_0-9]+)\",[[:space:]](?<container>[_A-Z]),[[:space:]](?<value>(\"[^\"]\"\|[^\",)]))")
		if err != nil {
		return err
		}
		declIn, err := os.ReadFile(path)
		if err != nil {
		return err
		}
		lines := strings.Split(string(declIn), "\n")
		var description string
		for _, line := range lines {
		if comment := commentRegexp.FindStringSubmatch(commentRegexp.FindString(line)); comment != nil {
		// Description is the text from any contiguous series of lines before a `flag()` call.
		description += fmt.Sprintf(" %s", strings.TrimSpace(comment[commentRegexp.SubexpIndex("comment")]))
		continue
		}
		matches := declRegexp.FindStringSubmatch(declRegexp.FindString(line))
		if matches == nil {
		// The line is neither a comment nor a `flag()` call.
		// Discard any description we have gathered and process the next line.
		description = ""
		continue
		}
		declValue := matches[declRegexp.SubexpIndex("value")]
		declName := matches[declRegexp.SubexpIndex("name")]
		container := rc_proto.Container(rc_proto.Container_value[matches[declRegexp.SubexpIndex("container")]])
		description = strings.TrimSpace(description)
		var namespace string
		var ok bool
		if namespace, ok = namespaceMap[declName]; !ok {
		namespace = defaultFlagNamespace
		}
		flagDeclaration := &rc_proto.FlagDeclaration{
		Name: proto.String(declName),
		Namespace: proto.String(namespace),
		Description: proto.String(description),
		Container: &container,
		}
		description = ""
		// Most build flags are `workflow: PREBUILT`.
		workflow := rc_proto.Workflow(rc_proto.Workflow_PREBUILT)
		switch {
		case declName == "RELEASE_ACONFIG_VALUE_SETS":
		rootAconfigModule = declValue[1 : len(declValue)-1]
		continue
		case strings.HasPrefix(declValue, "\""):
		// String values mean that the flag workflow is (most likely) either MANUAL or PREBUILT.
		declValue = declValue[1 : len(declValue)-1]
		flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{declValue}}
		for _, prefix := range manualFlagNamePrefixes {
		if strings.HasPrefix(declName, prefix) {
		workflow = rc_proto.Workflow(rc_proto.Workflow_MANUAL)
		break
		}
		}
		case declValue == "False" \|\| declValue == "True":
		// Boolean values are LAUNCH flags.
		flagDeclaration.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{declValue == "True"}}
		workflow = rc_proto.Workflow(rc_proto.Workflow_LAUNCH)
		case declValue == "None":
		// Use PREBUILT workflow with no initial value.
		default:
		fmt.Printf("%s: Unexpected value %s=%s\n", path, declName, declValue)
		}
		flagDeclaration.Workflow = &workflow
		if flagDeclaration != nil {
		declPath := filepath.Join(dir, "flag_declarations", fmt.Sprintf("%s.textproto", declName))
		err := WriteFile(declPath, flagDeclaration)
		if err != nil {
		return err
		}
		}
		}
		if rootAconfigModule != "" {
		rootProto := &rc_proto.ReleaseConfig{
		Name: proto.String("root"),
		AconfigValueSets: []string{rootAconfigModule},
		}
		return WriteFile(filepath.Join(dir, "release_configs", "root.textproto"), rootProto)
		}
		return nil
		}

		func ProcessBuildConfigs(dir, name string, paths []string, releaseProto *rc_proto.ReleaseConfig) error {
		valRegexp, err := regexp.Compile("[[:space:]]+value.\"(?<name>[A-Z_0-9]+)\",[[:space:]](?<value>[^,)])")
		if err != nil {
		return err
		}
		for _, path := range paths {
		fmt.Printf("Processing %s\n", path)
		valIn, err := os.ReadFile(path)
		if err != nil {
		fmt.Printf("%s: error: %v\n", path, err)
		return err
		}
		vals := valRegexp.FindAllString(string(valIn), -1)
		for _, val := range vals {
		matches := valRegexp.FindStringSubmatch(val)
		valValue := matches[valRegexp.SubexpIndex("value")]
		valName := matches[valRegexp.SubexpIndex("name")]
		flagValue := &rc_proto.FlagValue{
		Name: proto.String(valName),
		}
		switch {
		case valName == "RELEASE_ACONFIG_VALUE_SETS":
		flagValue = nil
		if releaseProto.AconfigValueSets == nil {
		releaseProto.AconfigValueSets = []string{}
		}
		releaseProto.AconfigValueSets = append(releaseProto.AconfigValueSets, valValue[1:len(valValue)-1])
		case strings.HasPrefix(valValue, "\""):
		valValue = valValue[1 : len(valValue)-1]
		flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_StringValue{valValue}}
		case valValue == "None":
		// nothing to do here.
		case valValue == "True":
		flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{true}}
		case valValue == "False":
		flagValue.Value = &rc_proto.Value{Val: &rc_proto.Value_BoolValue{false}}
		default:
		fmt.Printf("%s: Unexpected value %s=%s\n", path, valName, valValue)
		}
		if flagValue != nil {
		valPath := filepath.Join(dir, "flag_values", RenameNext(name), fmt.Sprintf("%s.textproto", valName))
		err := WriteFile(valPath, flagValue)
		if err != nil {
		return err
		}
		}
		}
		}
		return err
		}

		func ProcessReleaseConfigMap(dir string, descriptionMap map[string]string) error {
		path := filepath.Join(dir, "release_config_map.mk")
		if _, err := os.Stat(path); err != nil {
		fmt.Printf("%s not found, ignoring.\n", path)
		return nil
		} else {
		fmt.Printf("Processing %s\n", path)
		}
		configRegexp, err := regexp.Compile("^..call[[:space:]]+declare-release-config,[[:space:]]+(?<name>[_a-z0-0A-Z]+),[[:space:]]+(?<files>[^,])(,[[:space:]](?<inherits>.)\|[[:space:]])[)]$")
		if err != nil {
		return err
		}
		aliasRegexp, err := regexp.Compile("^..call[[:space:]]+alias-release-config,[[:space:]]+(?<name>[_a-z0-9A-Z]+),[[:space:]]+(?<target>[_a-z0-9A-Z]+)")
		if err != nil {
		return err
		}

		mapIn, err := os.ReadFile(path)
		if err != nil {
		return err
		}
		cleanDir := strings.TrimLeft(dir, "../")
		var defaultContainer rc_proto.Container
		switch {
		case strings.HasPrefix(cleanDir, "build/") \|\| cleanDir == "vendor/google_shared/build":
		defaultContainer = rc_proto.Container(rc_proto.Container_ALL)
		case cleanDir == "vendor/google/release":
		defaultContainer = rc_proto.Container(rc_proto.Container_ALL)
		default:
		defaultContainer = rc_proto.Container(rc_proto.Container_VENDOR)
		}
		releaseConfigMap := &rc_proto.ReleaseConfigMap{DefaultContainer: &defaultContainer}
		// If we find a description for the directory, include it.
		if description, ok := descriptionMap[cleanDir]; ok {
		releaseConfigMap.Description = proto.String(description)
		}
		lines := strings.Split(string(mapIn), "\n")
		for _, line := range lines {
		alias := aliasRegexp.FindStringSubmatch(aliasRegexp.FindString(line))
		if alias != nil {
		fmt.Printf("processing alias %s\n", line)
		name := alias[aliasRegexp.SubexpIndex("name")]
		target := alias[aliasRegexp.SubexpIndex("target")]
		if target == "next" {
		if RenameNext(target) != name {
		return fmt.Errorf("Unexpected name for next (%s)", RenameNext(target))
		}
		target, name = name, target
		}
		releaseConfigMap.Aliases = append(releaseConfigMap.Aliases,
		&rc_proto.ReleaseAlias{
		Name: proto.String(name),
		Target: proto.String(target),
		})
		}
		config := configRegexp.FindStringSubmatch(configRegexp.FindString(line))
		if config == nil {
		continue
		}
		name := config[configRegexp.SubexpIndex("name")]
		releaseConfig := &rc_proto.ReleaseConfig{
		Name: proto.String(RenameNext(name)),
		}
		configFiles := config[configRegexp.SubexpIndex("files")]
		files := strings.Split(strings.ReplaceAll(configFiles, "$(local_dir)", dir+"/"), " ")
		configInherits := config[configRegexp.SubexpIndex("inherits")]
		if len(configInherits) > 0 {
		releaseConfig.Inherits = strings.Split(configInherits, " ")
		}
		err := ProcessBuildConfigs(dir, name, files, releaseConfig)
		if err != nil {
		return err
		}

		releasePath := filepath.Join(dir, "release_configs", fmt.Sprintf("%s.textproto", RenameNext(name)))
		err = WriteFile(releasePath, releaseConfig)
		if err != nil {
		return err
		}
		}
		return WriteFile(filepath.Join(dir, "release_config_map.textproto"), releaseConfigMap)
		}

		func main() {
		var err error
		var top string
		var dirs rc_lib.StringList
		var namespacesFile string
		var descriptionsFile string

		flag.StringVar(&top, "top", ".", "path to top of workspace")
		flag.Var(&dirs, "dir", "directory to process, relative to the top of the workspace")
		flag.StringVar(&namespacesFile, "namespaces", "", "location of file with 'flag_name namespace' information")
		flag.StringVar(&descriptionsFile, "descriptions", "", "location of file with 'directory description' information")
		flag.Parse()

		if err = os.Chdir(top); err != nil {
		panic(err)
		}
		if len(dirs) == 0 {
		dirs = rc_lib.StringList{"build/release", "vendor/google_shared/build/release", "vendor/google/release"}
		}

		namespaceMap := make(map[string]string)
		if namespacesFile != "" {
		data, err := os.ReadFile(namespacesFile)
		if err != nil {
		panic(err)
		}
		for idx, line := range strings.Split(string(data), "\n") {
		fields := strings.Split(line, " ")
		if len(fields) > 2 {
		panic(fmt.Errorf("line %d: too many fields: %s", idx, line))
		}
		namespaceMap[fields[0]] = fields[1]
		}

		}

		descriptionMap := make(map[string]string)
		descriptionMap["build/release"] = "Published open-source flags and declarations"
		if descriptionsFile != "" {
		data, err := os.ReadFile(descriptionsFile)
		if err != nil {
		panic(err)
		}
		for _, line := range strings.Split(string(data), "\n") {
		if strings.TrimSpace(line) != "" {
		fields := strings.SplitN(line, " ", 2)
		descriptionMap[fields[0]] = fields[1]
		}
		}

		}

		for _, dir := range dirs {
		err = ProcessBuildFlags(dir, namespaceMap)
		if err != nil {
		panic(err)
		}

		err = ProcessReleaseConfigMap(dir, descriptionMap)
		if err != nil {
		panic(err)
		}
		}
		}