Emit and parse the product config variables from kati/make (f20c93af) · Commits · e / os / android_build

core/dumpconfig.mk

0 → 100644

+121 −0

Original line number	Diff line number	Diff line
		# Read and dump the product configuration.

		# Called from the product-config tool, not from the main build system.

		#
		# Ensure we are being called correctly
		#
		ifndef KATI
		$(warning Kati must be used to call dumpconfig.mk, not make.)
		$(error stopping)
		endif

		ifdef DEFAULT_GOAL
		$(warning Calling dumpconfig.mk from inside the make build system is not)
		$(warning supported. It is only meant to be called via kati by product-confing.)
		$(error stopping)
		endif

		ifndef TARGET_PRODUCT
		$(warning dumpconfig.mk requires TARGET_PRODUCT to be set)
		$(error stopping)
		endif

		ifndef TARGET_BUILD_VARIANT
		$(warning dumpconfig.mk requires TARGET_BUILD_VARIANT to be set)
		$(error stopping)
		endif

		ifneq (build/make/core/config.mk,$(wildcard build/make/core/config.mk))
		$(warning dumpconfig must be called from the root of the source tree)
		$(error stopping)
		endif

		ifeq (,$(DUMPCONFIG_FILE))
		$(warning dumpconfig requires DUMPCONFIG_FILE to be set)
		$(error stopping)
		endif

		# Before we do anything else output the format version.
		$(file > $(DUMPCONFIG_FILE),dumpconfig_version,1)
		$(file >> $(DUMPCONFIG_FILE),dumpconfig_file,$(DUMPCONFIG_FILE))

		# Default goal for dumpconfig
		dumpconfig:
		$(file >> $(DUMPCONFIG_FILE),*DONE*)
		@echo *DONE*

		# TODO(Remove): These need to be set externally
		OUT_DIR := out
		TMPDIR = /tmp/build-temp
		BUILD_DATETIME_FILE := $(OUT_DIR)/build_date.txt

		# Escape quotation marks for CSV, and wraps in quotation marks.
		define escape-for-csv
		"$(subst ","",$1)"
		endef

		# Args:
		# $(1): include stack
		define dump-import-start
		$(eval $(file >> $(DUMPCONFIG_FILE),import,$(strip $(1))))
		endef

		# Args:
		# $(1): include stack
		define dump-import-done
		$(eval $(file >> $(DUMPCONFIG_FILE),imported,$(strip $(1))))
		endef

		# Args:
		# $(1): Current file
		# $(2): Inherited file
		define dump-inherit
		$(eval $(file >> $(DUMPCONFIG_FILE),inherit,$(strip $(1)),$(strip $(2))))
		endef

		# Args:
		# $(1): Config phase (PRODUCT or DEVICE)
		# $(2): Root nodes to import
		# $(3): All variable names
		# $(4): Single-value variables
		define dump-product-var-names
		$(eval $(file >> $(DUMPCONFIG_FILE),phase,$(strip $(1)),$(strip $(2)))) \
		$(foreach var,$(3), \
		$(eval $(file >> $(DUMPCONFIG_FILE),var,$(if $(filter $(4),$(var)),single,list),$(var))) \
		)
		endef

		define dump-debug
		$(eval $(file >> $(DUMPCONFIG_FILE),debug,$(1)))
		endef

		# Skip these when dumping. They're not used and they cause a lot of noise in the dump.
		DUMPCONFIG_SKIP_VARS := \
		.VARIABLES \
		.KATI_SYMBOLS \
		1 \
		2 \
		LOCAL_PATH \
		MAKEFILE_LIST \
		PARENT_PRODUCT_FILES \
		current_mk \
		inherit_var \
		np \
		_node_import_context \
		_included \
		_include_stack \
		_in \
		_nic.%

		# Args:
		# $(1): Makefile that was included
		# $(2): block (before,import,after)
		define dump-config-vals
		$(foreach var,$(filter-out $(DUMPCONFIG_SKIP_VARS),$(.KATI_SYMBOLS)),\
		$(eval $(file >> $(DUMPCONFIG_FILE),val,$(call escape-for-csv,$(1)),$(2),$(call escape-for-csv,$(var)),$(call escape-for-csv,$($(var))),$(call escape-for-csv,$(KATI_variable_location $(var))))) \
		)
		endef

		include build/make/core/config.mk

core/node_fns.mk

+5 −0

Original line number	Diff line number	Diff line
		@@ -195,7 +195,11 @@ define _import-node
		$(call clear-var-list, $(3))
		$(eval LOCAL_PATH := $(patsubst %/,%,$(dir $(2))))
		$(eval MAKEFILE_LIST :=)
		$(call dump-import-start,$(_include_stack))
		$(call dump-config-vals,$(2),before)
		$(eval include $(2))
		$(call dump-import-done,$(_include_stack))
		$(call dump-config-vals,$(2),after)
		$(eval _included := $(filter-out $(2),$(MAKEFILE_LIST)))
		$(eval MAKEFILE_LIST :=)
		$(eval LOCAL_PATH :=)
		@@ -256,6 +260,7 @@ $(if \
		$(if $(_include_stack),$(eval $(error ASSERTION FAILED: _include_stack \
		should be empty here: $(_include_stack))),) \
		$(eval _include_stack := ) \
		$(call dump-product-var-names,$(1),$(2),$(3),$(4)) \
		$(call _import-nodes-inner,$(_node_import_context),$(_in),$(3),$(4)) \
		$(call move-var-list,$(_node_import_context).$(_in),$(1).$(_in),$(3)) \
		$(eval _node_import_context :=) \

core/product.mk

+3 −1

Original line number	Diff line number	Diff line
		@@ -460,7 +460,9 @@ define inherit-product
		$(eval current_mk := $(strip $(word 1,$(_include_stack)))) \
		$(eval inherit_var := PRODUCTS.$(current_mk).INHERITS_FROM) \
		$(eval $(inherit_var) := $(sort $($(inherit_var)) $(np))) \
		$(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk)))
		$(eval PARENT_PRODUCT_FILES := $(sort $(PARENT_PRODUCT_FILES) $(current_mk))) \
		$(call dump-inherit,$(strip $(word 1,$(_include_stack))),$(1)) \
		$(call dump-config-vals,$(current_mk),inherit)
		endef

		# Specifies a number of path prefixes, relative to PRODUCT_OUT, where the

tools/product_config/src/com/android/build/config/ConfigBase.java

0 → 100644

+78 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.build.config;

		import java.io.PrintStream;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.Map;
		import java.util.TreeMap;

		/**
		* Common parts between MakeConfig and the to-be-added GenericConfig, BazelConfig and SoongConfig.
		*/
		public class ConfigBase {
		protected String mPhase;
		protected List<String> mRootNodes;

		/**
		* The variables that are handled specially.
		*/
		protected final TreeMap<String, VarType> mProductVars = new TreeMap();

		/**
		* Whether a product config variable is a list or single-value variable.
		*/
		public enum VarType {
		LIST,
		SINGLE,
		UNKNOWN // For non-product vars
		}

		public void setPhase(String phase) {
		mPhase = phase;
		}

		public String getPhase() {
		return mPhase;
		}

		public void setRootNodes(List<String> filenames) {
		mRootNodes = new ArrayList(filenames);
		}

		public List<String> getRootNodes() {
		return mRootNodes;
		}

		public void addProductVar(String name, VarType type) {
		mProductVars.put(name, type);
		}

		public VarType getVarType(String name) {
		final VarType t = mProductVars.get(name);
		if (t != null) {
		return t;
		} else {
		return VarType.UNKNOWN;
		}
		}

		public boolean isProductVar(String name) {
		return mProductVars.get(name) != null;
		}
		}

tools/product_config/src/com/android/build/config/DumpConfigParser.java

0 → 100644

+304 −0

Original line number	Diff line number	Diff line

		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/


		package com.android.build.config;

		import java.io.IOException;
		import java.io.Reader;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.HashMap;
		import java.util.HashSet;
		import java.util.List;
		import java.util.Map;
		import java.util.Map;
		import java.util.regex.Pattern;

		/**
		* Parses the output of ckati building build/make/core/dumpconfig.mk.
		*
		* The format is as follows:
		* - All processed lines are colon (':') separated fields.
		* - Lines before the dumpconfig_version line are dropped for forward compatibility
		* - Lines where the first field is config_var describe variables declared in makefiles
		* (implemented by the dump-config-vals macro)
		* Field Description
		* 0 "config_var" row type
		* 1 Product makefile being processed
		* 2 The variable name
		* 3 The value of the variable
		* 4 The location of the variable, as best tracked by kati
		*/
		public class DumpConfigParser {
		private static final boolean DEBUG = true;

		private final Errors mErrors;
		private final String mFilename;
		private final Reader mReader;

		private final ArrayList<MakeConfig> mResults = new ArrayList();

		private static final Pattern LIST_SEPARATOR = Pattern.compile("\\s+");

		public class BuildPhase {

		}

		/**
		* Constructor.
		*/
		private DumpConfigParser(Errors errors, String filename, Reader reader) {
		mErrors = errors;
		mFilename = filename;
		mReader = reader;
		}

		/**
		* Parse the text into a list of MakeConfig objects.
		*/
		public static ArrayList<MakeConfig> parse(Errors errors, String filename, Reader reader)
		throws CsvParser.ParseException, IOException {
		DumpConfigParser parser = new DumpConfigParser(errors, filename, reader);
		parser.parseImpl();
		return parser.mResults;
		}

		/**
		* Parse the input.
		*/
		private void parseImpl() throws CsvParser.ParseException, IOException {
		final List<CsvParser.Line> lines = CsvParser.parse(mReader);
		final int lineCount = lines.size();
		int index = 0;

		int dumpconfigVersion = 0;

		// Ignore lines until until we get a dumpconfig_version line for forward compatibility.
		// In a previous life, this loop parsed from all of kati's stdout, not just the file
		// that dumpconfig.mk writes, but it's harmless to leave this loop in. It gives us a
		// little bit of flexibility which we probably won't need anyway, this tool probably
		// won't diverge from dumpconfig.mk anyway.
		for (; index < lineCount; index++) {
		final CsvParser.Line line = lines.get(index);
		final List<String> fields = line.getFields();

		if (matchLineType(line, "dumpconfig_version", 1)) {
		try {
		dumpconfigVersion = Integer.parseInt(fields.get(1));
		} catch (NumberFormatException ex) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Couldn't parse dumpconfig_version: " + fields.get(1));
		}
		break;
		}
		}

		// If we never saw dumpconfig_version, there's a problem with the command, so stop.
		if (dumpconfigVersion == 0) {
		mErrors.ERROR_DUMPCONFIG.fatal(
		new Position(mFilename),
		"Never saw a valid dumpconfig_version line.");
		}

		// Any lines before the start signal will be dropped. We create garbage objects
		// here to avoid having to check for null everywhere.
		MakeConfig makeConfig = new MakeConfig();
		MakeConfig.ConfigFile configFile = new MakeConfig.ConfigFile("<ignored>");
		MakeConfig.Block block = new MakeConfig.Block(MakeConfig.BlockType.UNSET);

		// Number of "phases" we've seen so far.
		for (; index < lineCount; index++) {
		final CsvParser.Line line = lines.get(index);
		final List<String> fields = line.getFields();
		final String lineType = fields.get(0);

		if (matchLineType(line, "phase", 2)) {
		makeConfig = new MakeConfig();
		makeConfig.setPhase(fields.get(1));
		makeConfig.setRootNodes(splitList(fields.get(2)));
		mResults.add(makeConfig);

		if (DEBUG) {
		System.out.println("PHASE:");
		System.out.println(" " + makeConfig.getPhase());
		System.out.println(" " + makeConfig.getRootNodes());
		}
		} else if (matchLineType(line, "var", 2)) {
		final MakeConfig.VarType type = "list".equals(fields.get(1))
		? MakeConfig.VarType.LIST : MakeConfig.VarType.SINGLE;
		makeConfig.addProductVar(fields.get(2), type);

		if (DEBUG) {
		System.out.println(" VAR: " + type + " " + fields.get(2));
		}
		} else if (matchLineType(line, "import", 1)) {
		final List<String> importStack = splitList(fields.get(1));
		if (importStack.size() == 0) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"'import' line with empty include stack.");
		continue;
		}

		// The beginning of importing a new file.
		configFile = new MakeConfig.ConfigFile(importStack.get(0));
		if (makeConfig.addConfigFile(configFile) != null) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Duplicate file imported in section: " + configFile.getFilename());
		}
		// We expect a Variable block next.
		block = new MakeConfig.Block(MakeConfig.BlockType.BEFORE);
		configFile.addBlock(block);

		if (DEBUG) {
		System.out.println(" IMPORT: " + configFile.getFilename());
		}
		} else if (matchLineType(line, "inherit", 2)) {
		final String currentFile = fields.get(1);
		final String inheritedFile = fields.get(2);
		if (!configFile.getFilename().equals(currentFile)) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Unexpected current file in 'inherit' line '" + currentFile
		+ "' while processing '" + configFile.getFilename() + "'");
		continue;
		}

		// There is already a file in progress, so add another var block to that.
		block = new MakeConfig.Block(MakeConfig.BlockType.INHERIT);
		block.setInheritedFile(inheritedFile);
		configFile.addBlock(block);

		if (DEBUG) {
		System.out.println(" INHERIT: " + inheritedFile);
		}
		} else if (matchLineType(line, "imported", 1)) {
		final List<String> importStack = splitList(fields.get(1));
		if (importStack.size() == 0) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"'imported' line with empty include stack.");
		continue;
		}
		final String currentFile = importStack.get(0);
		if (!configFile.getFilename().equals(currentFile)) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Unexpected current file in 'imported' line '" + currentFile
		+ "' while processing '" + configFile.getFilename() + "'");
		continue;
		}

		// There is already a file in progress, so add another var block to that.
		// This will be the last one, but will check that after parsing.
		block = new MakeConfig.Block(MakeConfig.BlockType.AFTER);
		configFile.addBlock(block);

		if (DEBUG) {
		System.out.println(" AFTER: " + currentFile);
		}
		} else if (matchLineType(line, "val", 5)) {
		final String productMakefile = fields.get(1);
		final MakeConfig.BlockType blockType = parseBlockType(line, fields.get(2));
		final String varName = fields.get(3);
		final String varValue = fields.get(4);
		final Position pos = Position.parse(fields.get(5));

		if (!productMakefile.equals(configFile.getFilename())) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Mismatched 'val' product makefile."
		+ " Expected: " + configFile.getFilename()
		+ " Saw: " + productMakefile);
		continue;
		}
		if (blockType == null) {
		continue;
		}
		if (blockType != block.getBlockType()) {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Mismatched 'val' block type."
		+ " Expected: " + block.getBlockType()
		+ " Saw: " + blockType);
		}

		// Add the value to the block in progress
		block.addValue(varName, new Str(pos, varValue));
		} else {
		if (DEBUG) {
		System.out.print("# ");
		for (int d = 0; d < fields.size(); d++) {
		System.out.print(fields.get(d));
		if (d != fields.size() - 1) {
		System.out.print(",");
		}
		}
		System.out.println();
		}
		}
		}
		}

		/**
		* Return true if the line type matches 'lineType' and there are at least 'fieldCount'
		* fields (not including the first field which is the line type).
		*/
		private boolean matchLineType(CsvParser.Line line, String lineType, int fieldCount) {
		final List<String> fields = line.getFields();
		if (!lineType.equals(fields.get(0))) {
		return false;
		}
		if (fields.size() < (fieldCount + 1)) {
		mErrors.WARNING_DUMPCONFIG.add(new Position(mFilename, line.getLine()),
		fields.get(0) + " line has " + fields.size() + " fields. Expected at least "
		+ (fieldCount + 1) + " fields.");
		return false;
		}
		return true;
		}

		/**
		* Split a string with space separated items (i.e. the make list format) into a List<String>.
		*/
		private static List<String> splitList(String text) {
		// Arrays.asList returns a fixed-length List, so we copy it into an ArrayList to not
		// propagate that surprise detail downstream.
		return new ArrayList(Arrays.asList(LIST_SEPARATOR.split(text.trim())));
		}

		/**
		* Parse a BockType or issue a warning if it can't be parsed.
		*/
		private MakeConfig.BlockType parseBlockType(CsvParser.Line line, String text) {
		if ("before".equals(text)) {
		return MakeConfig.BlockType.BEFORE;
		} else if ("inherit".equals(text)) {
		return MakeConfig.BlockType.INHERIT;
		} else if ("after".equals(text)) {
		return MakeConfig.BlockType.AFTER;
		} else {
		mErrors.WARNING_DUMPCONFIG.add(
		new Position(mFilename, line.getLine()),
		"Invalid block type: " + text);
		return null;
		}
		}
		}