Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 593bef71 authored by Joe Onorato's avatar Joe Onorato Committed by Gerrit Code Review
Browse files

Merge "Define release flags in starlark instead of make"

parents 9066ab52 386b3746

core/envsetup.mk

+8 −3
Original line number Diff line number Diff line
@@ -24,19 +24,24 @@ endef 
#$(warning $(call find_and_earlier,A B C,C))

#$(warning $(call find_and_earlier,A B C,D))



# Runs the starlark file given in $(1), and sets all the variables in its top-level

# Runs a starlark file, and sets all the variables in its top-level

# variables_to_export_to_make variable as make variables.

#

# In order to avoid running starlark every time the stamp file is checked, we use

# $(KATI_shell_no_rerun). Then, to make sure that we actually do rerun kati when

# modifying the starlark files, we add the starlark files to the kati stamp file with

# $(KATI_extra_file_deps).

#

# Arguments:

#  $(1): A single starlark file to use as the entrypoint

#  $(2): An optional list of starlark files to NOT include as kati dependencies.

#  $(3): An optional list of extra flags to pass to rbcrun

define run-starlark

$(eval _starlark_results := $(OUT_DIR)/starlark_results/$(subst /,_,$(1)).mk)

$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))

$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(3) $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))

$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Starlark failed to run))

$(eval include $(_starlark_results))

$(KATI_extra_file_deps $(LOADED_STARLARK_FILES))

$(KATI_extra_file_deps $(filter-out $(2),$(LOADED_STARLARK_FILES)))

$(eval LOADED_STARLARK_FILES :=)

$(eval _starlark_results :=)

endef

core/release_config.bzl

0 → 100644
+90 −0
Original line number Diff line number Diff line 
# Copyright (C) 2023 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



# Partitions that get build system flag summaries

_flag_partitions = [

    "product",

    "system",

    "system_ext",

    "vendor",

]



def _combine_dicts_no_duplicate_keys(dicts):

    result = {}

    for d in dicts:

        for k, v in d.items():

            if k in result:

                fail("Duplicate key: " + k)

            result[k] = v

    return result



def release_config(target_release, flag_definitions, config_maps, fail_if_no_release_config = True):

    result = {

        "_ALL_RELEASE_FLAGS": [flag.name for flag in flag_definitions],

    }

    all_flags = {}

    for flag in flag_definitions:

        if sorted(dir(flag)) != ["default", "name", "partitions"]:

            fail("Flag structs must contain 3 fields: name, partitions, and default")

        if not flag.partitions:

            fail("At least 1 partition is required")

        for partition in flag.partitions:

            if partition == "all":

                if len(flag.partitions) > 1:

                    fail("\"all\" can't be combined with other partitions: " + str(flag.partitions))

            elif partition not in _flag_partitions:

                fail("Invalid partition: " + flag.partition + ", allowed partitions: " + str(_flag_partitions))

        if not flag.name.startswith("RELEASE_"):

            fail("Release flag names must start with RELEASE_")

        if " " in flag.name or "\t" in flag.name or "\n" in flag.name:

            fail("Flag names must not contain whitespace.")

        if flag.name in all_flags:

            fail("Duplicate declaration of flag " + flag.name)

        all_flags[flag.name] = True



        default = flag.default

        if type(default) == "bool":

            default = "true" if default else ""



        result["_ALL_RELEASE_FLAGS." + flag.name + ".PARTITIONS"] = flag.partitions

        result["_ALL_RELEASE_FLAGS." + flag.name + ".DEFAULT"] = default

        result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = default



    # If TARGET_RELEASE is set, fail if there is no matching release config

    # If it isn't set, no release config files will be included and all flags

    # will get their default values.

    if target_release:

        config_map = _combine_dicts_no_duplicate_keys(config_maps)

        if target_release not in config_map:

            fail("No release config found for TARGET_RELEASE: " + target_release + ". Available releases are: " + str(config_map.keys()))

        release_config = config_map[target_release]

        if sorted(dir(release_config)) != ["flags", "release_version"]:

            fail("A release config must be a struct with a flags and release_version fields")

        result["_RELEASE_VERSION"] = release_config.release_version

        for flag in release_config.flags:

            if sorted(dir(flag)) != ["name", "value"]:

                fail("A flag must be a struct with name and value fields, got: " + str(sorted(dir(flag))))

            if flag.name not in all_flags:

                fail("Undeclared build flag: " + flag.name)

            value = flag.value

            if type(value) == "bool":

                value = "true" if value else ""

            result["_ALL_RELEASE_FLAGS." + flag.name + ".VALUE"] = value

    elif fail_if_no_release_config:

        fail("FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not")

    else:

        # No TARGET_RELEASE means release version 0

        result["_RELEASE_VERSION"] = 0



    return result

core/release_config.mk

+37 −180
Original line number Diff line number Diff line
@@ -12,78 +12,45 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



# Partitions that get build system flag summaries

_FLAG_PARTITIONS := system vendor system_ext product



# All possible release flags. Defined in the build_flags.mk files

# throughout the tree

_ALL_RELEASE_FLAGS :=



# -----------------------------------------------------------------

# Choose the flag files

# Do this first, because we're going to unset TARGET_RELEASE before

# including anyone, so they don't start making conditionals based on it.



# If this is a google source tree, restrict it to only the one file

# which has OWNERS control.  If it isn't let others define their own.

# TODO: Remove wildcard for build/release one when all branch manifests

# have updated.

config_map_files := $(wildcard build/release/release_config_map.mk) \

    $(if $(wildcard vendor/google/release/release_config_map.mk), \

        vendor/google/release/release_config_map.mk, \

flag_declaration_files := $(wildcard build/release/build_flags.bzl) \

    $(if $(wildcard vendor/google/release/build_flags.bzl), \

        vendor/google/release/build_flags.bzl, \

        $(sort \

            $(wildcard device/*/release/release_config_map.mk) \

            $(wildcard device/*/*/release/release_config_map.mk) \

            $(wildcard vendor/*/release/release_config_map.mk) \

            $(wildcard vendor/*/*/release/release_config_map.mk) \

            $(wildcard device/*/release/build_flags.bzl) \

            $(wildcard device/*/*/release/build_flags.bzl) \

            $(wildcard vendor/*/release/build_flags.bzl) \

            $(wildcard vendor/*/*/release/build_flags.bzl) \

        ) \

    )



# $1 config name

# $2 release config files

define declare-release-config

    $(eval # No duplicates)

    $(if $(filter $(_all_release_configs), $(strip $(1))), \

        $(error declare-release-config: config $(strip $(1)) declared in: $(_included) Previously declared here: $(_all_release_configs.$(strip $(1)).DECLARED_IN)) \

    )

    $(eval # Must have release config files)

    $(if $(strip $(2)),,  \

        $(error declare-release-config: config $(strip $(1)) must have release config files) \

    )

    $(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1))))

    $(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included))

    $(eval _all_release_configs.$(strip $(1)).FILES := $(strip $(2)))

endef



# Include the config map files

$(foreach f, $(config_map_files), \

    $(eval _included := $(f)) \

    $(eval include $(f)) \

config_map_files := $(wildcard build/release/release_config_map.bzl) \

    $(if $(wildcard vendor/google/release/release_config_map.bzl), \

        vendor/google/release/release_config_map.bzl, \

        $(sort \

            $(wildcard device/*/release/release_config_map.bzl) \

            $(wildcard device/*/*/release/release_config_map.bzl) \

            $(wildcard vendor/*/release/release_config_map.bzl) \

            $(wildcard vendor/*/*/release/release_config_map.bzl) \

        ) \

    )



# If TARGET_RELEASE is set, fail if there is no matching release config

# If it isn't set, no release config files will be included and all flags

# will get their default values.

ifneq ($(TARGET_RELEASE),)

ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),)

    $(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs))

else

    # Choose flag files

    # Don't sort this, use it in the order they gave us.

    _release_config_files := $(_all_release_configs.$(TARGET_RELEASE).FILES)

endif

else

# Useful for finding scripts etc that aren't passing or setting TARGET_RELEASE

ifneq ($(FAIL_IF_NO_RELEASE_CONFIG),)

    $(error FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not)

endif

_release_config_files :=

endif



# Unset variables so they can't use it

define declare-release-config

$(error declare-release-config can only be called from inside release_config_map.mk files)

endef

# Because starlark can't find files with $(wildcard), write an entrypoint starlark script that

# contains the result of the above wildcards for the starlark code to use.

filename_to_starlark=$(subst /,_,$(subst .,_,$(1)))

_c:=load("//build/make/core/release_config.bzl", "release_config")

_c+=$(foreach f,$(flag_declaration_files),$(newline)load("//$(f)", flags_$(call filename_to_starlark,$(f)) = "flags"))

_c+=$(foreach f,$(config_map_files),$(newline)load("//$(f)", config_maps_$(call filename_to_starlark,$(f)) = "config_maps"))

_c+=$(newline)all_flags = [] $(foreach f,$(flag_declaration_files),+ flags_$(call filename_to_starlark,$(f)))

_c+=$(newline)all_config_maps = [$(foreach f,$(config_map_files),config_maps_$(call filename_to_starlark,$(f))$(comma))]

_c+=$(newline)target_release = "$(TARGET_RELEASE)"

_c+=$(newline)fail_if_no_release_config = True if "$(FAIL_IF_NO_RELEASE_CONFIG)" else False

_c+=$(newline)variables_to_export_to_make = release_config(target_release, all_flags, all_config_maps, fail_if_no_release_config)

$(file >$(OUT_DIR)/release_config_entrypoint.bzl,$(_c))

_c:=

filename_to_starlark:=



# TODO: Remove this check after enough people have sourced lunch that we don't

# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023
@@ -96,116 +63,13 @@ TARGET_RELEASE:= 
endif

.KATI_READONLY := TARGET_RELEASE



$(foreach config, $(_all_release_configs), \

    $(eval _all_release_configs.$(config).DECLARED_IN:= ) \

    $(eval _all_release_configs.$(config).FILES:= ) \

)

_all_release_configs:=

config_map_files:=



# -----------------------------------------------------------------

# Declare the flags



# $1 partition(s)

# $2 flag name. Must start with RELEASE_

# $3 default. True or false

define declare-build-flag

    $(if $(filter-out all $(_FLAG_PARTITIONS), $(strip $(1))), \

        $(error declare-build-flag: invalid partitions: $(strip $(1))) \

    )

    $(if $(and $(filter all,$(strip $(1))),$(filter-out all, $(strip $(1)))), \

        $(error declare-build-flag: "all" can't be combined with other partitions: $(strip $(1))), \

        $(eval declare-build-flag.partition := $(_FLAG_PARTITIONS)) \

    )

    $(if $(filter-out RELEASE_%, $(strip $(2))), \

        $(error declare-build-flag: Release flag names must start with RELEASE_: $(strip $(2))) \

    )

    $(eval _ALL_RELEASE_FLAGS += $(strip $(2)))

    $(foreach partition, $(declare-build-flag.partition), \

        $(eval _ALL_RELEASE_FLAGS.PARTITIONS.$(partition) := $(sort \

            $(_ALL_RELEASE_FLAGS.PARTITIONS.$(partition)) $(strip $(2)))) \

    )

    $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).PARTITIONS := $(declare-build-flag.partition))

    $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DEFAULT := $(strip $(3)))

    $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DECLARED_IN := $(_included))

    $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).VALUE := $(strip $(3)))

    $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).SET_IN := $(_included))

    $(eval declare-build-flag.partition:=)

endef





# Choose the files

# If this is a google source tree, restrict it to only the one file

# which has OWNERS control.  If it isn't let others define their own.

flag_declaration_files := $(wildcard build/release/build_flags.mk) \

    $(if $(wildcard vendor/google/release/build_flags.mk), \

        vendor/google/release/build_flags.mk, \

        $(sort \

            $(wildcard device/*/release/build_flags.mk) \

            $(wildcard device/*/*/release/build_flags.mk) \

            $(wildcard vendor/*/release/build_flags.mk) \

            $(wildcard vendor/*/*/release/build_flags.mk) \

        ) \

    )



# Include the files

$(foreach f, $(flag_declaration_files), \

    $(eval _included := $(f)) \

    $(eval include $(f)) \

)



# Don't let anyone declare build flags after here

define declare-build-flag

$(error declare-build-flag can only be called from inside flag definition files.)

endef



# No more flags from here on

.KATI_READONLY := _ALL_RELEASE_FLAGS



# -----------------------------------------------------------------

# Set the flags



# $(1): Flag name. Must start with RELEASE_ and have been defined by declare-build-flag

# $(2): Value. True or false

define set-build-flag

    $(if $(filter-out $(_ALL_RELEASE_FLAGS), $(strip $(1))), \

        $(error set-build-flag: Undeclared build flag: $(strip $(1))) \

    )

    $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).VALUE := $(strip $(2)))

    $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).SET_IN := $(_included))

endef



# This writes directly to a file so that the version never exists in make for

# people to write conditionals upon.

define set-release-version

    $(eval _RELEASE_VERSION := $(strip $(1)))

endef



# Include the files (if there are any)

ifneq ($(strip $(_release_config_files)),)

    $(foreach f, $(_release_config_files), \

        $(eval _included := $(f)) \

        $(eval include $(f)) \

    )

else

    # No TARGET_RELEASE means release version 0

    $(call set-release-version, 0)

endif





ifeq ($(_RELEASE_VERSION)),)

    $(error No release config file called set-release-version. Included files were: $(_release_config_files))

endif



# Don't let anyone declare build flags after here

define set-build-flag

$(error set-build-flag can only be called from inside release config files.)

endef



# Don't let anyone set the release version after here

define set-release-version

$(error set-release-version can only be called from inside release config files.)

endef

# Exclude the entrypoint file as a dependency (by passing it as the 2nd argument) so that we don't

# rerun kati every build. Kati will replay the $(file) command that generates it every build,

# updating its timestamp.

#

# We also need to pass --allow_external_entrypoint to rbcrun in case the OUT_DIR is set to something

# outside of the source tree.

$(call run-starlark,$(OUT_DIR)/release_config_entrypoint.bzl,$(OUT_DIR)/release_config_entrypoint.bzl,--allow_external_entrypoint)



# Set the flag values, and don't allow any one to modify them.

$(foreach flag, $(_ALL_RELEASE_FLAGS), \
@@ -213,10 +77,3 @@ $(foreach flag, $(_ALL_RELEASE_FLAGS), \ 
    $(eval .KATI_READONLY := $(flag)) \

)



# -----------------------------------------------------------------

# Clear out vars

flag_declaration_files:=

flag_files:=

_included:=

_release_config_files:=

tools/rbcrun/host.go

+22 −15
Original line number Diff line number Diff line
@@ -34,9 +34,10 @@ const ( 
	ExecutionModeMake ExecutionMode = iota

)



const allowExternalEntrypointKey = "allowExternalEntrypoint"

const callerDirKey = "callerDir"

const shellKey = "shell"

const executionModeKey = "executionMode"

const shellKey = "shell"



type modentry struct {

	globals starlark.StringDict
@@ -64,7 +65,7 @@ var makeBuiltins starlark.StringDict = starlark.StringDict{ 


// Takes a module name (the first argument to the load() function) and returns the path

// it's trying to load, stripping out leading //, and handling leading :s.

func cleanModuleName(moduleName string, callerDir string) (string, error) {

func cleanModuleName(moduleName string, callerDir string, allowExternalPaths bool) (string, error) {

	if strings.Count(moduleName, ":") > 1 {

		return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)

	}
@@ -82,7 +83,7 @@ func cleanModuleName(moduleName string, callerDir string) (string, error) { 
	} else if strings.HasPrefix(moduleName, ":") {

		moduleName = moduleName[1:]

		localLoad = true

	} else {

	} else if !allowExternalPaths {

		return "", fmt.Errorf("load path must start with // or :")

	}
@@ -93,12 +94,14 @@ func cleanModuleName(moduleName string, callerDir string) (string, error) { 
	if filepath.Clean(moduleName) != moduleName {

		return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))

	}

	if !allowExternalPaths {

		if strings.HasPrefix(moduleName, "../") {

			return "", fmt.Errorf("load path must not start with ../: %s", moduleName)

		}

		if strings.HasPrefix(moduleName, "/") {

			return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)

		}

	}



	if localLoad {

		return filepath.Join(callerDir, moduleName), nil
@@ -114,17 +117,18 @@ func cleanModuleName(moduleName string, callerDir string) (string, error) { 
// bound to None if file is missing.

func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) {

	mode := thread.Local(executionModeKey).(ExecutionMode)

	allowExternalEntrypoint := thread.Local(allowExternalEntrypointKey).(bool)

	var defaultSymbol string

	mustLoad := true

	if mode == ExecutionModeRbc {

		pipePos := strings.LastIndex(module, "|")

		mustLoad = pipePos < 0

		if !mustLoad {

		if pipePos >= 0 {

			mustLoad = false

			defaultSymbol = module[pipePos+1:]

			module = module[:pipePos]

		}

	}

	modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))

	modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string), allowExternalEntrypoint)

	if err != nil {

		return nil, err

	}
@@ -155,9 +159,11 @@ func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) 
				childThread.SetLocal(testReporterKey, v)

			}



			// Only the entrypoint starlark file allows external loads.

			childThread.SetLocal(allowExternalEntrypointKey, false)

			childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))

			childThread.SetLocal(shellKey, thread.Local(shellKey))

			childThread.SetLocal(executionModeKey, mode)

			childThread.SetLocal(shellKey, thread.Local(shellKey))

			if mode == ExecutionModeRbc {

				globals, err := starlark.ExecFile(childThread, modulePath, nil, rbcBuiltins)

				e = &modentry{globals, err}
@@ -318,7 +324,7 @@ func log(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwa 
// * src is an optional source of bytes to use instead of filename

//   (it can be a string, or a byte array, or an io.Reader instance)

// Returns the top-level starlark variables, the list of starlark files loaded, and an error

func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringDict, []string, error) {

func Run(filename string, src interface{}, mode ExecutionMode, allowExternalEntrypoint bool) (starlark.StringDict, []string, error) {

	// NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,

	// which always uses /bin/sh to run the command

	shellPath := "/bin/sh"
@@ -347,7 +353,7 @@ func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringD 
		if err != nil {

			return nil, nil, err

		}

		if strings.HasPrefix(filename, "../") {

		if !allowExternalEntrypoint && strings.HasPrefix(filename, "../") {

			return nil, nil, fmt.Errorf("path could not be made relative to workspace root: %s", filename)

		}

	} else {
@@ -358,9 +364,10 @@ func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringD 
	moduleCache[filename] = nil



	var results starlark.StringDict

	mainThread.SetLocal(allowExternalEntrypointKey, allowExternalEntrypoint)

	mainThread.SetLocal(callerDirKey, filepath.Dir(filename))

	mainThread.SetLocal(shellKey, shellPath)

	mainThread.SetLocal(executionModeKey, mode)

	mainThread.SetLocal(shellKey, shellPath)

	if mode == ExecutionModeRbc {

		results, err = starlark.ExecFile(mainThread, filename, src, rbcBuiltins)

	} else if mode == ExecutionModeMake {

tools/rbcrun/host_test.go

+1 −0
Original line number Diff line number Diff line
@@ -125,6 +125,7 @@ func TestLoad(t *testing.T) { 
	if err := os.Chdir(filepath.Dir(dir)); err != nil {

		t.Fatal(err)

	}

	thread.SetLocal(allowExternalEntrypointKey, false)

	thread.SetLocal(callerDirKey, dir)

	thread.SetLocal(executionModeKey, ExecutionModeRbc)

	if _, err := starlark.ExecFile(thread, "testdata/load.star", nil, rbcBuiltins); err != nil {