Product config makefiles to Starlark converter (b051c4ed) · Commits · e / os / android_build_soong

androidmk/parser/make_strings.go

+9 −0

Original line number	Diff line number	Diff line
		@@ -278,6 +278,15 @@ func (ms *MakeString) ReplaceLiteral(input string, output string) {
		}
		}

		// If MakeString is $(var) after trimming, returns var
		func (ms MakeString) SingleVariable() (MakeString, bool) {
		if len(ms.Strings) != 2 \|\| strings.TrimSpace(ms.Strings[0]) != "" \|\|
		strings.TrimSpace(ms.Strings[1]) != "" {
		return nil, false
		}
		return ms.Variables[0].Name, true
		}

		func splitAnyN(s, sep string, n int) []string {
		ret := []string{}
		for n == -1 \|\| n > 1 {

androidmk/parser/parser.go

+2 −1

Original line number	Diff line number	Diff line
		@@ -216,13 +216,14 @@ func (p *parser) parseDirective() bool {
		// Nothing
		case "else":
		p.ignoreSpaces()
		if p.tok != '\n' {
		if p.tok != '\n' && p.tok != '#' {
		d = p.scanner.TokenText()
		p.accept(scanner.Ident)
		if d == "ifdef" \|\| d == "ifndef" \|\| d == "ifeq" \|\| d == "ifneq" {
		d = "el" + d
		p.ignoreSpaces()
		expression = p.parseExpression()
		expression.TrimRightSpaces()
		} else {
		p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d)
		}

mk2rbc/Android.bp

0 → 100644

+39 −0

Original line number	Diff line number	Diff line
		//
		// Copyright (C) 2021 The Android Open Source Project
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		blueprint_go_binary {
		name: "mk2rbc",
		srcs: ["cmd/mk2rbc.go"],
		deps: [
		"mk2rbc-lib",
		"androidmk-parser",
		],
		}

		bootstrap_go_package {
		name: "mk2rbc-lib",
		pkgPath: "android/soong/mk2rbc",
		srcs: [
		"android_products.go",
		"config_variables.go",
		"expr.go",
		"mk2rbc.go",
		"node.go",
		"soong_variables.go",
		"types.go",
		"variable.go",
		],
		deps: ["androidmk-parser"],
		}

mk2rbc/TODO

0 → 100644

+14 −0

Original line number	Diff line number	Diff line
		* Checking filter/filter-out results is incorrect if pattern contains '%'
		* Need heuristics to recognize that a variable is local. Propose to use lowercase.
		* Need heuristics for the local variable type. Propose '_list' suffix
		* Internal source tree has variables in the inherit-product macro argument. Handle it
		* Enumerate all environment variables that configuration files use.
		* Break mk2rbc.go into multiple files.
		* If variable's type is not yet known, try to divine it from the value assigned to it
		(it may be a variable of the known type, or a function result)
		* ifneq (,$(VAR)) should translate to
		if getattr(<>, "VAR", <default>):
		* Launcher file needs to have same suffix as the rest of the generated files
		* Implement $(shell) function
		* Write execution tests
		* Review all TODOs in mk2rbc.go
		No newline at end of file

mk2rbc/cmd/mk2rbc.go

0 → 100644

+498 −0

Original line number	Diff line number	Diff line
		// Copyright 2021 Google LLC
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		// The application to convert product configuration makefiles to Starlark.
		// Converts either given list of files (and optionally the dependent files
		// of the same kind), or all all product configuration makefiles in the
		// given source tree.
		// Previous version of a converted file can be backed up.
		// Optionally prints detailed statistics at the end.
		package main

		import (
		"flag"
		"fmt"
		"io/ioutil"
		"os"
		"path/filepath"
		"regexp"
		"runtime/debug"
		"sort"
		"strings"
		"time"

		"android/soong/androidmk/parser"
		"android/soong/mk2rbc"
		)

		var (
		rootDir = flag.String("root", ".", "the value of // for load paths")
		// TODO(asmundak): remove this option once there is a consensus on suffix
		suffix = flag.String("suffix", ".rbc", "generated files' suffix")
		dryRun = flag.Bool("dry_run", false, "dry run")
		recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
		mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
		warn = flag.Bool("warnings", false, "warn about partially failed conversions")
		verbose = flag.Bool("v", false, "print summary")
		errstat = flag.Bool("error_stat", false, "print error statistics")
		traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
		// TODO(asmundak): this option is for debugging
		allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
		outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
		launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
		printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
		traceCalls = flag.Bool("trace_calls", false, "trace function calls")
		)

		func init() {
		// Poor man's flag aliasing: works, but the usage string is ugly and
		// both flag and its alias can be present on the command line
		flagAlias := func(target string, alias string) {
		if f := flag.Lookup(target); f != nil {
		flag.Var(f.Value, alias, "alias for --"+f.Name)
		return
		}
		quit("cannot alias unknown flag " + target)
		}
		flagAlias("suffix", "s")
		flagAlias("root", "d")
		flagAlias("dry_run", "n")
		flagAlias("convert_dependents", "r")
		flagAlias("warnings", "w")
		flagAlias("error_stat", "e")
		}

		var backupSuffix string
		var tracedVariables []string
		var errorLogger = errorsByType{data: make(map[string]datum)}

		func main() {
		flag.Usage = func() {
		cmd := filepath.Base(os.Args[0])
		fmt.Fprintf(flag.CommandLine.Output(),
		"Usage: %[1]s flags file...\n"+
		"or: %[1]s flags --launcher=PATH PRODUCT\n", cmd)
		flag.PrintDefaults()
		}
		flag.Parse()

		// Delouse
		if *suffix == ".mk" {
		quit("cannot use .mk as generated file suffix")
		}
		if *suffix == "" {
		quit("suffix cannot be empty")
		}
		if *outputTop != "" {
		if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
		quit(err)
		}
		s, err := filepath.Abs(*outputTop)
		if err != nil {
		quit(err)
		}
		*outputTop = s
		}
		if *allInSource && len(flag.Args()) > 0 {
		quit("file list cannot be specified when -all is present")
		}
		if allInSource && launcher != "" {
		quit("--all and --launcher are mutually exclusive")
		}

		// Flag-driven adjustments
		if (*suffix)[0] != '.' {
		suffix = "." + suffix
		}
		if *mode == "backup" {
		backupSuffix = time.Now().Format("20060102150405")
		}
		if *traceVar != "" {
		tracedVariables = strings.Split(*traceVar, ",")
		}

		// Find out global variables
		getConfigVariables()
		getSoongVariables()

		if *printProductConfigMap {
		productConfigMap := buildProductConfigMap()
		var products []string
		for p := range productConfigMap {
		products = append(products, p)
		}
		sort.Strings(products)
		for _, p := range products {
		fmt.Println(p, productConfigMap[p])
		}
		os.Exit(0)
		}
		if len(flag.Args()) == 0 {
		flag.Usage()
		}
		// Convert!
		ok := true
		if *launcher != "" {
		if len(flag.Args()) != 1 {
		quit(fmt.Errorf("a launcher can be generated only for a single product"))
		}
		product := flag.Args()[0]
		productConfigMap := buildProductConfigMap()
		path, found := productConfigMap[product]
		if !found {
		quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
		product))
		}
		ok = convertOne(path) && ok
		err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
		if err != nil {
		fmt.Fprintf(os.Stderr, "%s:%s", path, err)
		ok = false
		}

		} else {
		files := flag.Args()
		if *allInSource {
		productConfigMap := buildProductConfigMap()
		for _, path := range productConfigMap {
		files = append(files, path)
		}
		}
		for _, mkFile := range files {
		ok = convertOne(mkFile) && ok
		}
		}

		printStats()
		if *errstat {
		errorLogger.printStatistics()
		}
		if !ok {
		os.Exit(1)
		}
		}

		func quit(s interface{}) {
		fmt.Fprintln(os.Stderr, s)
		os.Exit(2)
		}

		func buildProductConfigMap() map[string]string {
		const androidProductsMk = "AndroidProducts.mk"
		// Build the list of AndroidProducts.mk files: it's
		// build/make/target/product/AndroidProducts.mk plus
		// device/**/AndroidProducts.mk
		targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
		if _, err := os.Stat(targetAndroidProductsFile); err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
		targetAndroidProductsFile, err, *rootDir)
		}
		productConfigMap := make(map[string]string)
		if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
		}
		_ = filepath.Walk(filepath.Join(*rootDir, "device"),
		func(path string, info os.FileInfo, err error) error {
		if info.IsDir() \|\| filepath.Base(path) != androidProductsMk {
		return nil
		}
		if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
		// Keep going, we want to find all such errors in a single run
		}
		return nil
		})
		return productConfigMap
		}

		func getConfigVariables() {
		path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
		if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
		quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
		err, *rootDir))
		}
		}

		// Implements mkparser.Scope, to be used by mkparser.Value.Value()
		type fileNameScope struct {
		mk2rbc.ScopeBase
		}

		func (s fileNameScope) Get(name string) string {
		if name != "BUILD_SYSTEM" {
		return fmt.Sprintf("$(%s)", name)
		}
		return filepath.Join(*rootDir, "build", "make", "core")
		}

		func getSoongVariables() {
		path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
		err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
		if err != nil {
		quit(err)
		}
		}

		var converted = make(map[string]*mk2rbc.StarlarkScript)

		//goland:noinspection RegExpRepeatedSpace
		var cpNormalizer = regexp.MustCompile(
		"# Copyright \$C\$ 20.. The Android Open Source Project")

		const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
		const copyright = `#
		# Copyright (C) 20xx The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.
		#
		`

		// Convert a single file.
		// Write the result either to the same directory, to the same place in
		// the output hierarchy, or to the stdout.
		// Optionally, recursively convert the files this one includes by
		// $(call inherit-product) or an include statement.
		func convertOne(mkFile string) (ok bool) {
		if v, ok := converted[mkFile]; ok {
		return v != nil
		}
		converted[mkFile] = nil
		defer func() {
		if r := recover(); r != nil {
		ok = false
		fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
		}
		}()

		mk2starRequest := mk2rbc.Request{
		MkFile: mkFile,
		Reader: nil,
		RootDir: *rootDir,
		OutputDir: *outputTop,
		OutputSuffix: *suffix,
		TracedVariables: tracedVariables,
		TraceCalls: *traceCalls,
		WarnPartialSuccess: *warn,
		}
		if *errstat {
		mk2starRequest.ErrorLogger = errorLogger
		}
		ss, err := mk2rbc.Convert(mk2starRequest)
		if err != nil {
		fmt.Fprintln(os.Stderr, mkFile, ": ", err)
		return false
		}
		script := ss.String()
		outputPath := outputFilePath(mkFile)

		if *dryRun {
		fmt.Printf("==== %s ====\n", outputPath)
		// Print generated script after removing the copyright header
		outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
		fmt.Println(strings.TrimPrefix(outText, copyright))
		} else {
		if err := maybeBackup(outputPath); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return false
		}
		if err := writeGenerated(outputPath, script); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return false
		}
		}
		ok = true
		if *recurse {
		for _, sub := range ss.SubConfigFiles() {
		// File may be absent if it is a conditional load
		if _, err := os.Stat(sub); os.IsNotExist(err) {
		continue
		}
		ok = convertOne(sub) && ok
		}
		}
		converted[mkFile] = ss
		return ok
		}

		// Optionally saves the previous version of the generated file
		func maybeBackup(filename string) error {
		stat, err := os.Stat(filename)
		if os.IsNotExist(err) {
		return nil
		}
		if !stat.Mode().IsRegular() {
		return fmt.Errorf("%s exists and is not a regular file", filename)
		}
		switch *mode {
		case "backup":
		return os.Rename(filename, filename+backupSuffix)
		case "write":
		return os.Remove(filename)
		default:
		return fmt.Errorf("%s already exists, use --mode option", filename)
		}
		}

		func outputFilePath(mkFile string) string {
		path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
		if *outputTop != "" {
		path = filepath.Join(*outputTop, path)
		}
		return path
		}

		func writeGenerated(path string, contents string) error {
		if err := os.MkdirAll(filepath.Dir(path), os.ModeDir\|os.ModePerm); err != nil {
		return err
		}
		if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
		return err
		}
		return nil
		}

		func printStats() {
		var sortedFiles []string
		if !warn && !verbose {
		return
		}
		for p := range converted {
		sortedFiles = append(sortedFiles, p)
		}
		sort.Strings(sortedFiles)

		nOk, nPartial, nFailed := 0, 0, 0
		for _, f := range sortedFiles {
		if converted[f] == nil {
		nFailed++
		} else if converted[f].HasErrors() {
		nPartial++
		} else {
		nOk++
		}
		}
		if *warn {
		if nPartial > 0 {
		fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
		for _, f := range sortedFiles {
		if ss := converted[f]; ss != nil && ss.HasErrors() {
		fmt.Fprintln(os.Stderr, " ", f)
		}
		}
		}

		if nFailed > 0 {
		fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
		for _, f := range sortedFiles {
		if converted[f] == nil {
		fmt.Fprintln(os.Stderr, " ", f)
		}
		}
		}
		}
		if *verbose {
		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
		fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
		}
		}

		type datum struct {
		count int
		formattingArgs []string
		}

		type errorsByType struct {
		data map[string]datum
		}

		func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
		v, exists := ebt.data[message]
		if exists {
		v.count++
		} else {
		v = datum{1, nil}
		}
		if strings.Contains(message, "%s") {
		var newArg1 string
		if len(args) == 0 {
		panic(fmt.Errorf(`%s has %%s but args are missing`, message))
		}
		newArg1 = fmt.Sprint(args[0])
		if message == "unsupported line" {
		newArg1 = node.Dump()
		} else if message == "unsupported directive %s" {
		if newArg1 == "include" \|\| newArg1 == "-include" {
		newArg1 = node.Dump()
		}
		}
		v.formattingArgs = append(v.formattingArgs, newArg1)
		}
		ebt.data[message] = v
		}

		func (ebt errorsByType) printStatistics() {
		if len(ebt.data) > 0 {
		fmt.Fprintln(os.Stderr, "Error counts:")
		}
		for message, data := range ebt.data {
		if len(data.formattingArgs) == 0 {
		fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
		continue
		}
		itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
		fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
		fmt.Fprintln(os.Stderr, " ", itemsByFreq)
		}
		}

		func stringsWithFreq(items []string, topN int) (string, int) {
		freq := make(map[string]int)
		for _, item := range items {
		freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
		}
		var sorted []string
		for item := range freq {
		sorted = append(sorted, item)
		}
		sort.Slice(sorted, func(i int, j int) bool {
		return freq[sorted[i]] > freq[sorted[j]]
		})
		sep := ""
		res := ""
		for i, item := range sorted {
		if i >= topN {
		res += " ..."
		break
		}
		count := freq[item]
		if count > 1 {
		res += fmt.Sprintf("%s%s(%d)", sep, item, count)
		} else {
		res += fmt.Sprintf("%s%s", sep, item)
		}
		sep = ", "
		}
		return res, len(sorted)
		}