Clean up the API for release config in starlark (d6df20a0) · Commits · e / os / android_build

core/Makefile

+11 −9

Original line number	Diff line number	Diff line
		@@ -27,26 +27,28 @@ $(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1)))
		$(eval $(strip $(1)): PRIVATE_FLAG_NAMES := $(strip $(2)))
		$(strip $(1)):
		mkdir -p $$(dir $$(PRIVATE_OUT))
		( \
		echo '{' ; \
		echo 'flags: [' ; \
		echo '{' > $$(PRIVATE_OUT)
		echo '"flags": [' >> $$(PRIVATE_OUT)
		$$(foreach flag, $$(PRIVATE_FLAG_NAMES), \
		( \
		printf ' { "name": "%s", "value": "%s", ' \
		'$$(flag)' \
		'$$(_ALL_RELEASE_FLAGS.$$(flag).VALUE)' \
		; \
		printf '"set": "%s", "default": "%s", "declared": "%s", }' \
		printf '"set": "%s", "default": "%s", "declared": "%s" }' \
		'$$(_ALL_RELEASE_FLAGS.$$(flag).SET_IN)' \
		'$$(_ALL_RELEASE_FLAGS.$$(flag).DEFAULT)' \
		'$$(_ALL_RELEASE_FLAGS.$$(flag).DECLARED_IN)' \
		; \
		printf '$$(if $$(filter $$(lastword $$(PRIVATE_FLAG_NAMES)),$$(flag)),,$$(comma))\n' ; \
		) \
		echo "]" ; \
		echo "}" \
		) >> $$(PRIVATE_OUT)
		) >> $$(PRIVATE_OUT) \
		)
		echo "]" >> $$(PRIVATE_OUT)
		echo "}" >> $$(PRIVATE_OUT)
		endef

		_FLAG_PARTITIONS := product system system_ext vendor

		$(foreach partition, $(_FLAG_PARTITIONS), \
		$(eval BUILD_FLAG_SUMMARIES.$(partition) \
		:= $(TARGET_OUT_FLAGS)/$(partition)/etc/build_flags.json) \

core/release_config.bzl

+91 −61

Original line number	Diff line number	Diff line
		@@ -20,71 +20,101 @@ _flag_partitions = [
		"vendor",
		]

		def _combine_dicts_no_duplicate_keys(dicts):
		result = {}
		for d in dicts:
		for k, v in d.items():
		if k in result:
		fail("Duplicate key: " + k)
		result[k] = v
		return result
		ALL = ["all"]
		PRODUCT = ["product"]
		SYSTEM = ["system"]
		SYSTEM_EXT = ["system_ext"]
		VENDOR = ["vendor"]

		def release_config(target_release, flag_definitions, config_maps, fail_if_no_release_config = True):
		result = {
		"_ALL_RELEASE_FLAGS": [flag.name for flag in flag_definitions],
		}
		all_flags = {}
		for flag in flag_definitions:
		if sorted(dir(flag)) != ["default", "name", "partitions"]:
		fail("Flag structs must contain 3 fields: name, partitions, and default")
		if not flag.partitions:
		_valid_types = ["NoneType", "bool", "list", "string", "int"]

		def flag(name, partitions, default):
		"Declare a flag."
		if not partitions:
		fail("At least 1 partition is required")
		for partition in flag.partitions:
		if not name.startswith("RELEASE_"):
		fail("Release flag names must start with RELEASE_")
		if " " in name or "\t" in name or "\n" in name:
		fail("Flag names must not contain whitespace: \"" + name + "\"")
		for partition in partitions:
		if partition == "all":
		if len(flag.partitions) > 1:
		fail("\"all\" can't be combined with other partitions: " + str(flag.partitions))
		if len(partitions) > 1:
		fail("\"all\" can't be combined with other partitions: " + str(partitions))
		elif partition not in _flag_partitions:
		fail("Invalid partition: " + flag.partition + ", allowed partitions: " + str(_flag_partitions))
		if not flag.name.startswith("RELEASE_"):
		fail("Release flag names must start with RELEASE_")
		if " " in flag.name or "\t" in flag.name or "\n" in flag.name:
		fail("Flag names must not contain whitespace.")
		if flag.name in all_flags:
		fail("Duplicate declaration of flag " + flag.name)
		all_flags[flag.name] = True
		fail("Invalid partition: " + partition + ", allowed partitions: " +
		str(_flag_partitions))
		if type(default) not in _valid_types:
		fail("Invalid type of default for flag \"" + name + "\" (" + type(default) + ")")
		return {
		"name": name,
		"partitions": partitions,
		"default": default,
		}

		default = flag.default
		if type(default) == "bool":
		default = "true" if default else ""
		def value(name, value):
		"Define the flag value for a particular configuration."
		return {
		"name": name,
		"value": value,
		}

		result["_ALL_RELEASE_FLAGS." + flag.name + ".PARTITIONS"] = flag.partitions
		result["_ALL_RELEASE_FLAGS." + flag.name + ".DEFAULT"] = default
		result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = default
		def _format_value(val):
		"Format the starlark type correctly for make"
		if type(val) == "NoneType":
		return ""
		elif type(val) == "bool":
		return "true" if val else ""
		else:
		return val

		# If TARGET_RELEASE is set, fail if there is no matching release config
		# If it isn't set, no release config files will be included and all flags
		# will get their default values.
		if target_release:
		config_map = _combine_dicts_no_duplicate_keys(config_maps)
		if target_release not in config_map:
		fail("No release config found for TARGET_RELEASE: " + target_release + ". Available releases are: " + str(config_map.keys()))
		release_config = config_map[target_release]
		if sorted(dir(release_config)) != ["flags", "release_version"]:
		fail("A release config must be a struct with a flags and release_version fields")
		result["_RELEASE_VERSION"] = release_config.release_version
		for flag in release_config.flags:
		if sorted(dir(flag)) != ["name", "value"]:
		fail("A flag must be a struct with name and value fields, got: " + str(sorted(dir(flag))))
		if flag.name not in all_flags:
		fail("Undeclared build flag: " + flag.name)
		value = flag.value
		if type(value) == "bool":
		value = "true" if value else ""
		result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = value
		elif fail_if_no_release_config:
		fail("FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not")
		def release_config(all_flags, all_values):
		"Return the make variables that should be set for this release config."

		# Validate flags
		flag_names = []
		for flag in all_flags:
		if flag["name"] in flag_names:
		fail(flag["declared_in"] + ": Duplicate declaration of flag " + flag["name"])
		flag_names.append(flag["name"])

		# Record which flags go on which partition
		partitions = {}
		for flag in all_flags:
		for partition in flag["partitions"]:
		if partition == "all":
		for partition in _flag_partitions:
		partitions.setdefault(partition, []).append(flag["name"])
		else:
		partitions.setdefault(partition, []).append(flag["name"])

		# Validate values
		values = {}
		for value in all_values:
		if value["name"] not in flag_names:
		fail(value["set_in"] + ": Value set for undeclared build flag: " + value["name"])
		values[value["name"]] = value

		# Collect values
		result = {
		"_ALL_RELEASE_FLAGS": sorted(flag_names),
		}
		for partition, names in partitions.items():
		result["_ALL_RELEASE_FLAGS.PARTITIONS." + partition] = names
		for flag in all_flags:
		if flag["name"] in values:
		val = values[flag["name"]]["value"]
		set_in = values[flag["name"]]["set_in"]
		if type(val) not in _valid_types:
		fail("Invalid type of value for flag \"" + flag["name"] + "\" (" + type(val) + ")")
		else:
		# No TARGET_RELEASE means release version 0
		result["_RELEASE_VERSION"] = 0
		val = flag["default"]
		set_in = flag["declared_in"]
		val = _format_value(val)
		result[flag["name"]] = val
		result["_ALL_RELEASE_FLAGS." + flag["name"] + ".PARTITIONS"] = flag["partitions"]
		result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DEFAULT"] = _format_value(flag["default"])
		result["_ALL_RELEASE_FLAGS." + flag["name"] + ".VALUE"] = val
		result["_ALL_RELEASE_FLAGS." + flag["name"] + ".DECLARED_IN"] = flag["declared_in"]
		result["_ALL_RELEASE_FLAGS." + flag["name"] + ".SET_IN"] = set_in

		return result

core/release_config.mk

+108 −34

Original line number	Diff line number	Diff line
		@@ -12,6 +12,104 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.


		# -----------------------------------------------------------------
		# Choose the flag files
		# -----------------------------------------------------------------
		# Do this first, because we're going to unset TARGET_RELEASE before
		# including anyone, so they don't start making conditionals based on it.
		# This logic is in make because starlark doesn't understand optional
		# vendor files.

		# If this is a google source tree, restrict it to only the one file
		# which has OWNERS control. If it isn't let others define their own.
		# TODO: Remove wildcard for build/release one when all branch manifests
		# have updated.
		config_map_files := $(wildcard build/release/release_config_map.mk) \
		$(if $(wildcard vendor/google/release/release_config_map.mk), \
		vendor/google/release/release_config_map.mk, \
		$(sort \
		$(wildcard device/*/release/release_config_map.mk) \
		$(wildcard device///release/release_config_map.mk) \
		$(wildcard vendor/*/release/release_config_map.mk) \
		$(wildcard vendor///release/release_config_map.mk) \
		) \
		)

		# $1 config name
		# $2 release config files
		define declare-release-config
		$(eval # No duplicates)
		$(if $(filter $(_all_release_configs), $(strip $(1))), \
		$(error declare-release-config: config $(strip $(1)) declared in: $(_included) Previously declared here: $(_all_release_configs.$(strip $(1)).DECLARED_IN)) \
		)
		$(eval # Must have release config files)
		$(if $(strip $(2)),, \
		$(error declare-release-config: config $(strip $(1)) must have release config files) \
		)
		$(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1))))
		$(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included))
		$(eval _all_release_configs.$(strip $(1)).FILES := $(strip $(2)))
		endef

		# Include the config map files
		$(foreach f, $(config_map_files), \
		$(eval _included := $(f)) \
		$(eval include $(f)) \
		)

		# If TARGET_RELEASE is set, fail if there is no matching release config
		# If it isn't set, no release config files will be included and all flags
		# will get their default values.
		ifneq ($(TARGET_RELEASE),)
		ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),)
		$(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs))
		else
		# Choose flag files
		# Don't sort this, use it in the order they gave us.
		flag_value_files := $(_all_release_configs.$(TARGET_RELEASE).FILES)
		endif
		else
		# Useful for finding scripts etc that aren't passing or setting TARGET_RELEASE
		ifneq ($(FAIL_IF_NO_RELEASE_CONFIG),)
		$(error FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not)
		endif
		flag_value_files :=
		endif

		# Unset variables so they can't use them
		define declare-release-config
		$(error declare-release-config can only be called from inside release_config_map.mk files)
		endef

		# TODO: Remove this check after enough people have sourced lunch that we don't
		# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023
		ifneq ($(CALLED_FROM_SETUP),true)
		define TARGET_RELEASE
		$(error TARGET_RELEASE may not be accessed directly. Use individual flags.)
		endef
		else
		TARGET_RELEASE:=
		endif
		.KATI_READONLY := TARGET_RELEASE


		$(foreach config, $(_all_release_configs), \
		$(eval _all_release_configs.$(config).DECLARED_IN:= ) \
		$(eval _all_release_configs.$(config).FILES:= ) \
		)
		_all_release_configs:=
		config_map_files:=


		# -----------------------------------------------------------------
		# Flag declarations and values
		# -----------------------------------------------------------------
		# This part is in starlark. We generate a root starlark file that loads
		# all of the flags declaration files that we found, and the flag_value_files
		# that we chose from the config map above. Then we run that, and load the
		# results of that into the make environment.

		# If this is a google source tree, restrict it to only the one file
		# which has OWNERS control. If it isn't let others define their own.
		# TODO: Remove wildcard for build/release one when all branch manifests
		@@ -26,43 +124,25 @@ flag_declaration_files := $(wildcard build/release/build_flags.bzl) \
		$(wildcard vendor///release/build_flags.bzl) \
		) \
		)
		config_map_files := $(wildcard build/release/release_config_map.bzl) \
		$(if $(wildcard vendor/google/release/release_config_map.bzl), \
		vendor/google/release/release_config_map.bzl, \
		$(sort \
		$(wildcard device/*/release/release_config_map.bzl) \
		$(wildcard device///release/release_config_map.bzl) \
		$(wildcard vendor/*/release/release_config_map.bzl) \
		$(wildcard vendor///release/release_config_map.bzl) \
		) \
		)


		# Because starlark can't find files with $(wildcard), write an entrypoint starlark script that
		# contains the result of the above wildcards for the starlark code to use.
		filename_to_starlark=$(subst /,_,$(subst .,_,$(1)))
		_c:=load("//build/make/core/release_config.bzl", "release_config")
		_c+=$(foreach f,$(flag_declaration_files),$(newline)load("//$(f)", flags_$(call filename_to_starlark,$(f)) = "flags"))
		_c+=$(foreach f,$(config_map_files),$(newline)load("//$(f)", config_maps_$(call filename_to_starlark,$(f)) = "config_maps"))
		_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ flags_$(call filename_to_starlark,$(f)))
		_c+=$(newline)all_config_maps = [$(foreach f,$(config_map_files),config_maps_$(call filename_to_starlark,$(f))$(comma))]
		_c+=$(newline)target_release = "$(TARGET_RELEASE)"
		_c+=$(newline)fail_if_no_release_config = True if "$(FAIL_IF_NO_RELEASE_CONFIG)" else False
		_c+=$(newline)variables_to_export_to_make = release_config(target_release, all_flags, all_config_maps, fail_if_no_release_config)
		_c+=$(newline)def add(d, k, v):
		_c+=$(newline)$(space)d = dict(d)
		_c+=$(newline)$(space)d[k] = v
		_c+=$(newline)$(space)return d
		_c+=$(foreach f,$(flag_declaration_files),$(newline)load("$(f)", flags_$(call filename_to_starlark,$(f)) = "flags"))
		_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ [add(x, "declared_in", "$(f)") for x in flags_$(call filename_to_starlark,$(f))])
		_c+=$(foreach f,$(flag_value_files),$(newline)load("//$(f)", values_$(call filename_to_starlark,$(f)) = "values"))
		_c+=$(newline)all_values = [] $(foreach f,$(flag_value_files),+ [add(x, "set_in", "$(f)") for x in values_$(call filename_to_starlark,$(f))])
		_c+=$(newline)variables_to_export_to_make = release_config(all_flags, all_values)
		$(file >$(OUT_DIR)/release_config_entrypoint.bzl,$(_c))
		_c:=
		filename_to_starlark:=

		# TODO: Remove this check after enough people have sourced lunch that we don't
		# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023
		ifneq ($(CALLED_FROM_SETUP),true)
		define TARGET_RELEASE
		$(error TARGET_RELEASE may not be accessed directly. Use individual flags.)
		endef
		else
		TARGET_RELEASE:=
		endif
		.KATI_READONLY := TARGET_RELEASE

		# Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't
		# rerun kati every build. Kati will replay the $(file) command that generates it every build,
		# updating its timestamp.
		@@ -71,9 +151,3 @@ endif
		# outside of the source tree.
		$(call run-starlark,$(OUT_DIR)/release_config_entrypoint.bzl,$(OUT_DIR)/release_config_entrypoint.bzl,--allow_external_entrypoint)

		# Set the flag values, and don't allow any one to modify them.
		$(foreach flag, $(_ALL_RELEASE_FLAGS), \
		$(eval $(flag) := $(_ALL_RELEASE_FLAGS.$(flag).VALUE)) \
		$(eval .KATI_READONLY := $(flag)) \
		)