Loading androidmk/parser/make_strings.go +9 −0 Original line number Diff line number Diff line Loading @@ -278,6 +278,15 @@ func (ms *MakeString) ReplaceLiteral(input string, output string) { } } // If MakeString is $(var) after trimming, returns var func (ms *MakeString) SingleVariable() (*MakeString, bool) { if len(ms.Strings) != 2 || strings.TrimSpace(ms.Strings[0]) != "" || strings.TrimSpace(ms.Strings[1]) != "" { return nil, false } return ms.Variables[0].Name, true } func splitAnyN(s, sep string, n int) []string { ret := []string{} for n == -1 || n > 1 { Loading androidmk/parser/parser.go +2 −1 Original line number Diff line number Diff line Loading @@ -216,13 +216,14 @@ func (p *parser) parseDirective() bool { // Nothing case "else": p.ignoreSpaces() if p.tok != '\n' { if p.tok != '\n' && p.tok != '#' { d = p.scanner.TokenText() p.accept(scanner.Ident) if d == "ifdef" || d == "ifndef" || d == "ifeq" || d == "ifneq" { d = "el" + d p.ignoreSpaces() expression = p.parseExpression() expression.TrimRightSpaces() } else { p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d) } Loading mk2rbc/Android.bp 0 → 100644 +39 −0 Original line number Diff line number Diff line // // Copyright (C) 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. blueprint_go_binary { name: "mk2rbc", srcs: ["cmd/mk2rbc.go"], deps: [ "mk2rbc-lib", "androidmk-parser", ], } bootstrap_go_package { name: "mk2rbc-lib", pkgPath: "android/soong/mk2rbc", srcs: [ "android_products.go", "config_variables.go", "expr.go", "mk2rbc.go", "node.go", "soong_variables.go", "types.go", "variable.go", ], deps: ["androidmk-parser"], } mk2rbc/TODO 0 → 100644 +14 −0 Original line number Diff line number Diff line * Checking filter/filter-out results is incorrect if pattern contains '%' * Need heuristics to recognize that a variable is local. Propose to use lowercase. * Need heuristics for the local variable type. Propose '_list' suffix * Internal source tree has variables in the inherit-product macro argument. Handle it * Enumerate all environment variables that configuration files use. * Break mk2rbc.go into multiple files. * If variable's type is not yet known, try to divine it from the value assigned to it (it may be a variable of the known type, or a function result) * ifneq (,$(VAR)) should translate to if getattr(<>, "VAR", <default>): * Launcher file needs to have same suffix as the rest of the generated files * Implement $(shell) function * Write execution tests * Review all TODOs in mk2rbc.go No newline at end of file mk2rbc/cmd/mk2rbc.go 0 → 100644 +498 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // The application to convert product configuration makefiles to Starlark. // Converts either given list of files (and optionally the dependent files // of the same kind), or all all product configuration makefiles in the // given source tree. // Previous version of a converted file can be backed up. // Optionally prints detailed statistics at the end. package main import ( "flag" "fmt" "io/ioutil" "os" "path/filepath" "regexp" "runtime/debug" "sort" "strings" "time" "android/soong/androidmk/parser" "android/soong/mk2rbc" ) var ( rootDir = flag.String("root", ".", "the value of // for load paths") // TODO(asmundak): remove this option once there is a consensus on suffix suffix = flag.String("suffix", ".rbc", "generated files' suffix") dryRun = flag.Bool("dry_run", false, "dry run") recurse = flag.Bool("convert_dependents", false, "convert all dependent files") mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`) warn = flag.Bool("warnings", false, "warn about partially failed conversions") verbose = flag.Bool("v", false, "print summary") errstat = flag.Bool("error_stat", false, "print error statistics") traceVar = flag.String("trace", "", "comma-separated list of variables to trace") // TODO(asmundak): this option is for debugging allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //") outputTop = flag.String("outdir", "", "write output files into this directory hierarchy") launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_") printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit") traceCalls = flag.Bool("trace_calls", false, "trace function calls") ) func init() { // Poor man's flag aliasing: works, but the usage string is ugly and // both flag and its alias can be present on the command line flagAlias := func(target string, alias string) { if f := flag.Lookup(target); f != nil { flag.Var(f.Value, alias, "alias for --"+f.Name) return } quit("cannot alias unknown flag " + target) } flagAlias("suffix", "s") flagAlias("root", "d") flagAlias("dry_run", "n") flagAlias("convert_dependents", "r") flagAlias("warnings", "w") flagAlias("error_stat", "e") } var backupSuffix string var tracedVariables []string var errorLogger = errorsByType{data: make(map[string]datum)} func main() { flag.Usage = func() { cmd := filepath.Base(os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), "Usage: %[1]s flags file...\n"+ "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd) flag.PrintDefaults() } flag.Parse() // Delouse if *suffix == ".mk" { quit("cannot use .mk as generated file suffix") } if *suffix == "" { quit("suffix cannot be empty") } if *outputTop != "" { if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil { quit(err) } s, err := filepath.Abs(*outputTop) if err != nil { quit(err) } *outputTop = s } if *allInSource && len(flag.Args()) > 0 { quit("file list cannot be specified when -all is present") } if *allInSource && *launcher != "" { quit("--all and --launcher are mutually exclusive") } // Flag-driven adjustments if (*suffix)[0] != '.' { *suffix = "." + *suffix } if *mode == "backup" { backupSuffix = time.Now().Format("20060102150405") } if *traceVar != "" { tracedVariables = strings.Split(*traceVar, ",") } // Find out global variables getConfigVariables() getSoongVariables() if *printProductConfigMap { productConfigMap := buildProductConfigMap() var products []string for p := range productConfigMap { products = append(products, p) } sort.Strings(products) for _, p := range products { fmt.Println(p, productConfigMap[p]) } os.Exit(0) } if len(flag.Args()) == 0 { flag.Usage() } // Convert! ok := true if *launcher != "" { if len(flag.Args()) != 1 { quit(fmt.Errorf("a launcher can be generated only for a single product")) } product := flag.Args()[0] productConfigMap := buildProductConfigMap() path, found := productConfigMap[product] if !found { quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product", product)) } ok = convertOne(path) && ok err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path))) if err != nil { fmt.Fprintf(os.Stderr, "%s:%s", path, err) ok = false } } else { files := flag.Args() if *allInSource { productConfigMap := buildProductConfigMap() for _, path := range productConfigMap { files = append(files, path) } } for _, mkFile := range files { ok = convertOne(mkFile) && ok } } printStats() if *errstat { errorLogger.printStatistics() } if !ok { os.Exit(1) } } func quit(s interface{}) { fmt.Fprintln(os.Stderr, s) os.Exit(2) } func buildProductConfigMap() map[string]string { const androidProductsMk = "AndroidProducts.mk" // Build the list of AndroidProducts.mk files: it's // build/make/target/product/AndroidProducts.mk plus // device/**/AndroidProducts.mk targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) if _, err := os.Stat(targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", targetAndroidProductsFile, err, *rootDir) } productConfigMap := make(map[string]string) if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) } _ = filepath.Walk(filepath.Join(*rootDir, "device"), func(path string, info os.FileInfo, err error) error { if info.IsDir() || filepath.Base(path) != androidProductsMk { return nil } if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) // Keep going, we want to find all such errors in a single run } return nil }) return productConfigMap } func getConfigVariables() { path := filepath.Join(*rootDir, "build", "make", "core", "product.mk") if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil { quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)", err, *rootDir)) } } // Implements mkparser.Scope, to be used by mkparser.Value.Value() type fileNameScope struct { mk2rbc.ScopeBase } func (s fileNameScope) Get(name string) string { if name != "BUILD_SYSTEM" { return fmt.Sprintf("$(%s)", name) } return filepath.Join(*rootDir, "build", "make", "core") } func getSoongVariables() { path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk") err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables) if err != nil { quit(err) } } var converted = make(map[string]*mk2rbc.StarlarkScript) //goland:noinspection RegExpRepeatedSpace var cpNormalizer = regexp.MustCompile( "# Copyright \\(C\\) 20.. The Android Open Source Project") const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project" const copyright = `# # Copyright (C) 20xx The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ` // Convert a single file. // Write the result either to the same directory, to the same place in // the output hierarchy, or to the stdout. // Optionally, recursively convert the files this one includes by // $(call inherit-product) or an include statement. func convertOne(mkFile string) (ok bool) { if v, ok := converted[mkFile]; ok { return v != nil } converted[mkFile] = nil defer func() { if r := recover(); r != nil { ok = false fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack()) } }() mk2starRequest := mk2rbc.Request{ MkFile: mkFile, Reader: nil, RootDir: *rootDir, OutputDir: *outputTop, OutputSuffix: *suffix, TracedVariables: tracedVariables, TraceCalls: *traceCalls, WarnPartialSuccess: *warn, } if *errstat { mk2starRequest.ErrorLogger = errorLogger } ss, err := mk2rbc.Convert(mk2starRequest) if err != nil { fmt.Fprintln(os.Stderr, mkFile, ": ", err) return false } script := ss.String() outputPath := outputFilePath(mkFile) if *dryRun { fmt.Printf("==== %s ====\n", outputPath) // Print generated script after removing the copyright header outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright) fmt.Println(strings.TrimPrefix(outText, copyright)) } else { if err := maybeBackup(outputPath); err != nil { fmt.Fprintln(os.Stderr, err) return false } if err := writeGenerated(outputPath, script); err != nil { fmt.Fprintln(os.Stderr, err) return false } } ok = true if *recurse { for _, sub := range ss.SubConfigFiles() { // File may be absent if it is a conditional load if _, err := os.Stat(sub); os.IsNotExist(err) { continue } ok = convertOne(sub) && ok } } converted[mkFile] = ss return ok } // Optionally saves the previous version of the generated file func maybeBackup(filename string) error { stat, err := os.Stat(filename) if os.IsNotExist(err) { return nil } if !stat.Mode().IsRegular() { return fmt.Errorf("%s exists and is not a regular file", filename) } switch *mode { case "backup": return os.Rename(filename, filename+backupSuffix) case "write": return os.Remove(filename) default: return fmt.Errorf("%s already exists, use --mode option", filename) } } func outputFilePath(mkFile string) string { path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix if *outputTop != "" { path = filepath.Join(*outputTop, path) } return path } func writeGenerated(path string, contents string) error { if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil { return err } if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { return err } return nil } func printStats() { var sortedFiles []string if !*warn && !*verbose { return } for p := range converted { sortedFiles = append(sortedFiles, p) } sort.Strings(sortedFiles) nOk, nPartial, nFailed := 0, 0, 0 for _, f := range sortedFiles { if converted[f] == nil { nFailed++ } else if converted[f].HasErrors() { nPartial++ } else { nOk++ } } if *warn { if nPartial > 0 { fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n") for _, f := range sortedFiles { if ss := converted[f]; ss != nil && ss.HasErrors() { fmt.Fprintln(os.Stderr, " ", f) } } } if nFailed > 0 { fmt.Fprintf(os.Stderr, "Conversion failed for files:\n") for _, f := range sortedFiles { if converted[f] == nil { fmt.Fprintln(os.Stderr, " ", f) } } } } if *verbose { fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk) fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial) fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed) } } type datum struct { count int formattingArgs []string } type errorsByType struct { data map[string]datum } func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) { v, exists := ebt.data[message] if exists { v.count++ } else { v = datum{1, nil} } if strings.Contains(message, "%s") { var newArg1 string if len(args) == 0 { panic(fmt.Errorf(`%s has %%s but args are missing`, message)) } newArg1 = fmt.Sprint(args[0]) if message == "unsupported line" { newArg1 = node.Dump() } else if message == "unsupported directive %s" { if newArg1 == "include" || newArg1 == "-include" { newArg1 = node.Dump() } } v.formattingArgs = append(v.formattingArgs, newArg1) } ebt.data[message] = v } func (ebt errorsByType) printStatistics() { if len(ebt.data) > 0 { fmt.Fprintln(os.Stderr, "Error counts:") } for message, data := range ebt.data { if len(data.formattingArgs) == 0 { fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message) continue } itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30) fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count) fmt.Fprintln(os.Stderr, " ", itemsByFreq) } } func stringsWithFreq(items []string, topN int) (string, int) { freq := make(map[string]int) for _, item := range items { freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++ } var sorted []string for item := range freq { sorted = append(sorted, item) } sort.Slice(sorted, func(i int, j int) bool { return freq[sorted[i]] > freq[sorted[j]] }) sep := "" res := "" for i, item := range sorted { if i >= topN { res += " ..." break } count := freq[item] if count > 1 { res += fmt.Sprintf("%s%s(%d)", sep, item, count) } else { res += fmt.Sprintf("%s%s", sep, item) } sep = ", " } return res, len(sorted) } Loading
androidmk/parser/make_strings.go +9 −0 Original line number Diff line number Diff line Loading @@ -278,6 +278,15 @@ func (ms *MakeString) ReplaceLiteral(input string, output string) { } } // If MakeString is $(var) after trimming, returns var func (ms *MakeString) SingleVariable() (*MakeString, bool) { if len(ms.Strings) != 2 || strings.TrimSpace(ms.Strings[0]) != "" || strings.TrimSpace(ms.Strings[1]) != "" { return nil, false } return ms.Variables[0].Name, true } func splitAnyN(s, sep string, n int) []string { ret := []string{} for n == -1 || n > 1 { Loading
androidmk/parser/parser.go +2 −1 Original line number Diff line number Diff line Loading @@ -216,13 +216,14 @@ func (p *parser) parseDirective() bool { // Nothing case "else": p.ignoreSpaces() if p.tok != '\n' { if p.tok != '\n' && p.tok != '#' { d = p.scanner.TokenText() p.accept(scanner.Ident) if d == "ifdef" || d == "ifndef" || d == "ifeq" || d == "ifneq" { d = "el" + d p.ignoreSpaces() expression = p.parseExpression() expression.TrimRightSpaces() } else { p.errorf("expected ifdef/ifndef/ifeq/ifneq, found %s", d) } Loading
mk2rbc/Android.bp 0 → 100644 +39 −0 Original line number Diff line number Diff line // // Copyright (C) 2021 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. blueprint_go_binary { name: "mk2rbc", srcs: ["cmd/mk2rbc.go"], deps: [ "mk2rbc-lib", "androidmk-parser", ], } bootstrap_go_package { name: "mk2rbc-lib", pkgPath: "android/soong/mk2rbc", srcs: [ "android_products.go", "config_variables.go", "expr.go", "mk2rbc.go", "node.go", "soong_variables.go", "types.go", "variable.go", ], deps: ["androidmk-parser"], }
mk2rbc/TODO 0 → 100644 +14 −0 Original line number Diff line number Diff line * Checking filter/filter-out results is incorrect if pattern contains '%' * Need heuristics to recognize that a variable is local. Propose to use lowercase. * Need heuristics for the local variable type. Propose '_list' suffix * Internal source tree has variables in the inherit-product macro argument. Handle it * Enumerate all environment variables that configuration files use. * Break mk2rbc.go into multiple files. * If variable's type is not yet known, try to divine it from the value assigned to it (it may be a variable of the known type, or a function result) * ifneq (,$(VAR)) should translate to if getattr(<>, "VAR", <default>): * Launcher file needs to have same suffix as the rest of the generated files * Implement $(shell) function * Write execution tests * Review all TODOs in mk2rbc.go No newline at end of file
mk2rbc/cmd/mk2rbc.go 0 → 100644 +498 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // The application to convert product configuration makefiles to Starlark. // Converts either given list of files (and optionally the dependent files // of the same kind), or all all product configuration makefiles in the // given source tree. // Previous version of a converted file can be backed up. // Optionally prints detailed statistics at the end. package main import ( "flag" "fmt" "io/ioutil" "os" "path/filepath" "regexp" "runtime/debug" "sort" "strings" "time" "android/soong/androidmk/parser" "android/soong/mk2rbc" ) var ( rootDir = flag.String("root", ".", "the value of // for load paths") // TODO(asmundak): remove this option once there is a consensus on suffix suffix = flag.String("suffix", ".rbc", "generated files' suffix") dryRun = flag.Bool("dry_run", false, "dry run") recurse = flag.Bool("convert_dependents", false, "convert all dependent files") mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`) warn = flag.Bool("warnings", false, "warn about partially failed conversions") verbose = flag.Bool("v", false, "print summary") errstat = flag.Bool("error_stat", false, "print error statistics") traceVar = flag.String("trace", "", "comma-separated list of variables to trace") // TODO(asmundak): this option is for debugging allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //") outputTop = flag.String("outdir", "", "write output files into this directory hierarchy") launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_") printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit") traceCalls = flag.Bool("trace_calls", false, "trace function calls") ) func init() { // Poor man's flag aliasing: works, but the usage string is ugly and // both flag and its alias can be present on the command line flagAlias := func(target string, alias string) { if f := flag.Lookup(target); f != nil { flag.Var(f.Value, alias, "alias for --"+f.Name) return } quit("cannot alias unknown flag " + target) } flagAlias("suffix", "s") flagAlias("root", "d") flagAlias("dry_run", "n") flagAlias("convert_dependents", "r") flagAlias("warnings", "w") flagAlias("error_stat", "e") } var backupSuffix string var tracedVariables []string var errorLogger = errorsByType{data: make(map[string]datum)} func main() { flag.Usage = func() { cmd := filepath.Base(os.Args[0]) fmt.Fprintf(flag.CommandLine.Output(), "Usage: %[1]s flags file...\n"+ "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd) flag.PrintDefaults() } flag.Parse() // Delouse if *suffix == ".mk" { quit("cannot use .mk as generated file suffix") } if *suffix == "" { quit("suffix cannot be empty") } if *outputTop != "" { if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil { quit(err) } s, err := filepath.Abs(*outputTop) if err != nil { quit(err) } *outputTop = s } if *allInSource && len(flag.Args()) > 0 { quit("file list cannot be specified when -all is present") } if *allInSource && *launcher != "" { quit("--all and --launcher are mutually exclusive") } // Flag-driven adjustments if (*suffix)[0] != '.' { *suffix = "." + *suffix } if *mode == "backup" { backupSuffix = time.Now().Format("20060102150405") } if *traceVar != "" { tracedVariables = strings.Split(*traceVar, ",") } // Find out global variables getConfigVariables() getSoongVariables() if *printProductConfigMap { productConfigMap := buildProductConfigMap() var products []string for p := range productConfigMap { products = append(products, p) } sort.Strings(products) for _, p := range products { fmt.Println(p, productConfigMap[p]) } os.Exit(0) } if len(flag.Args()) == 0 { flag.Usage() } // Convert! ok := true if *launcher != "" { if len(flag.Args()) != 1 { quit(fmt.Errorf("a launcher can be generated only for a single product")) } product := flag.Args()[0] productConfigMap := buildProductConfigMap() path, found := productConfigMap[product] if !found { quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product", product)) } ok = convertOne(path) && ok err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path))) if err != nil { fmt.Fprintf(os.Stderr, "%s:%s", path, err) ok = false } } else { files := flag.Args() if *allInSource { productConfigMap := buildProductConfigMap() for _, path := range productConfigMap { files = append(files, path) } } for _, mkFile := range files { ok = convertOne(mkFile) && ok } } printStats() if *errstat { errorLogger.printStatistics() } if !ok { os.Exit(1) } } func quit(s interface{}) { fmt.Fprintln(os.Stderr, s) os.Exit(2) } func buildProductConfigMap() map[string]string { const androidProductsMk = "AndroidProducts.mk" // Build the list of AndroidProducts.mk files: it's // build/make/target/product/AndroidProducts.mk plus // device/**/AndroidProducts.mk targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) if _, err := os.Stat(targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", targetAndroidProductsFile, err, *rootDir) } productConfigMap := make(map[string]string) if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) } _ = filepath.Walk(filepath.Join(*rootDir, "device"), func(path string, info os.FileInfo, err error) error { if info.IsDir() || filepath.Base(path) != androidProductsMk { return nil } if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) // Keep going, we want to find all such errors in a single run } return nil }) return productConfigMap } func getConfigVariables() { path := filepath.Join(*rootDir, "build", "make", "core", "product.mk") if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil { quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)", err, *rootDir)) } } // Implements mkparser.Scope, to be used by mkparser.Value.Value() type fileNameScope struct { mk2rbc.ScopeBase } func (s fileNameScope) Get(name string) string { if name != "BUILD_SYSTEM" { return fmt.Sprintf("$(%s)", name) } return filepath.Join(*rootDir, "build", "make", "core") } func getSoongVariables() { path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk") err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables) if err != nil { quit(err) } } var converted = make(map[string]*mk2rbc.StarlarkScript) //goland:noinspection RegExpRepeatedSpace var cpNormalizer = regexp.MustCompile( "# Copyright \\(C\\) 20.. The Android Open Source Project") const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project" const copyright = `# # Copyright (C) 20xx The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ` // Convert a single file. // Write the result either to the same directory, to the same place in // the output hierarchy, or to the stdout. // Optionally, recursively convert the files this one includes by // $(call inherit-product) or an include statement. func convertOne(mkFile string) (ok bool) { if v, ok := converted[mkFile]; ok { return v != nil } converted[mkFile] = nil defer func() { if r := recover(); r != nil { ok = false fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack()) } }() mk2starRequest := mk2rbc.Request{ MkFile: mkFile, Reader: nil, RootDir: *rootDir, OutputDir: *outputTop, OutputSuffix: *suffix, TracedVariables: tracedVariables, TraceCalls: *traceCalls, WarnPartialSuccess: *warn, } if *errstat { mk2starRequest.ErrorLogger = errorLogger } ss, err := mk2rbc.Convert(mk2starRequest) if err != nil { fmt.Fprintln(os.Stderr, mkFile, ": ", err) return false } script := ss.String() outputPath := outputFilePath(mkFile) if *dryRun { fmt.Printf("==== %s ====\n", outputPath) // Print generated script after removing the copyright header outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright) fmt.Println(strings.TrimPrefix(outText, copyright)) } else { if err := maybeBackup(outputPath); err != nil { fmt.Fprintln(os.Stderr, err) return false } if err := writeGenerated(outputPath, script); err != nil { fmt.Fprintln(os.Stderr, err) return false } } ok = true if *recurse { for _, sub := range ss.SubConfigFiles() { // File may be absent if it is a conditional load if _, err := os.Stat(sub); os.IsNotExist(err) { continue } ok = convertOne(sub) && ok } } converted[mkFile] = ss return ok } // Optionally saves the previous version of the generated file func maybeBackup(filename string) error { stat, err := os.Stat(filename) if os.IsNotExist(err) { return nil } if !stat.Mode().IsRegular() { return fmt.Errorf("%s exists and is not a regular file", filename) } switch *mode { case "backup": return os.Rename(filename, filename+backupSuffix) case "write": return os.Remove(filename) default: return fmt.Errorf("%s already exists, use --mode option", filename) } } func outputFilePath(mkFile string) string { path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix if *outputTop != "" { path = filepath.Join(*outputTop, path) } return path } func writeGenerated(path string, contents string) error { if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil { return err } if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { return err } return nil } func printStats() { var sortedFiles []string if !*warn && !*verbose { return } for p := range converted { sortedFiles = append(sortedFiles, p) } sort.Strings(sortedFiles) nOk, nPartial, nFailed := 0, 0, 0 for _, f := range sortedFiles { if converted[f] == nil { nFailed++ } else if converted[f].HasErrors() { nPartial++ } else { nOk++ } } if *warn { if nPartial > 0 { fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n") for _, f := range sortedFiles { if ss := converted[f]; ss != nil && ss.HasErrors() { fmt.Fprintln(os.Stderr, " ", f) } } } if nFailed > 0 { fmt.Fprintf(os.Stderr, "Conversion failed for files:\n") for _, f := range sortedFiles { if converted[f] == nil { fmt.Fprintln(os.Stderr, " ", f) } } } } if *verbose { fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk) fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial) fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed) } } type datum struct { count int formattingArgs []string } type errorsByType struct { data map[string]datum } func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) { v, exists := ebt.data[message] if exists { v.count++ } else { v = datum{1, nil} } if strings.Contains(message, "%s") { var newArg1 string if len(args) == 0 { panic(fmt.Errorf(`%s has %%s but args are missing`, message)) } newArg1 = fmt.Sprint(args[0]) if message == "unsupported line" { newArg1 = node.Dump() } else if message == "unsupported directive %s" { if newArg1 == "include" || newArg1 == "-include" { newArg1 = node.Dump() } } v.formattingArgs = append(v.formattingArgs, newArg1) } ebt.data[message] = v } func (ebt errorsByType) printStatistics() { if len(ebt.data) > 0 { fmt.Fprintln(os.Stderr, "Error counts:") } for message, data := range ebt.data { if len(data.formattingArgs) == 0 { fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message) continue } itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30) fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count) fmt.Fprintln(os.Stderr, " ", itemsByFreq) } } func stringsWithFreq(items []string, topN int) (string, int) { freq := make(map[string]int) for _, item := range items { freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++ } var sorted []string for item := range freq { sorted = append(sorted, item) } sort.Slice(sorted, func(i int, j int) bool { return freq[sorted[i]] > freq[sorted[j]] }) sep := "" res := "" for i, item := range sorted { if i >= topN { res += " ..." break } count := freq[item] if count > 1 { res += fmt.Sprintf("%s%s(%d)", sep, item, count) } else { res += fmt.Sprintf("%s%s", sep, item) } sep = ", " } return res, len(sorted) }
