Merge "Handle foreach expressions in mk2rbc" (a6628d24) · Commits · e / os / android_build_soong

mk2rbc/expr.go

+243 −4

Original line number	Diff line number	Diff line
		@@ -31,6 +31,12 @@ type starlarkExpr interface {
		// Emit the code to copy the expression, otherwise we will end up
		// with source and target pointing to the same list.
		emitListVarCopy(gctx *generationContext)
		// Return the expression, calling the transformer func for
		// every expression in the tree. If the transformer func returns non-nil,
		// its result is used in place of the expression it was called with in the
		// resulting expression. The resulting starlarkExpr will contain as many
		// of the same objects from the original expression as possible.
		transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr
		}

		func maybeString(expr starlarkExpr) (string, bool) {
		@@ -62,6 +68,14 @@ func (s stringLiteralExpr) emitListVarCopy(gctx generationContext) {
		s.emit(gctx)
		}

		func (s *stringLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(s); replacement != nil {
		return replacement
		} else {
		return s
		}
		}

		// Integer literal
		type intLiteralExpr struct {
		literal int
		@@ -85,6 +99,14 @@ func (s intLiteralExpr) emitListVarCopy(gctx generationContext) {
		s.emit(gctx)
		}

		func (s *intLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(s); replacement != nil {
		return replacement
		} else {
		return s
		}
		}

		// Boolean literal
		type boolLiteralExpr struct {
		literal bool
		@@ -110,6 +132,14 @@ func (b boolLiteralExpr) emitListVarCopy(gctx generationContext) {
		b.emit(gctx)
		}

		func (b *boolLiteralExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(b); replacement != nil {
		return replacement
		} else {
		return b
		}
		}

		// interpolateExpr represents Starlark's interpolation operator <string> % list
		// we break <string> into a list of chunks, i.e., "first%second%third" % (X, Y)
		// will have chunks = ["first", "second", "third"] and args = [X, Y]
		@@ -190,6 +220,19 @@ func (xi interpolateExpr) emitListVarCopy(gctx generationContext) {
		xi.emit(gctx)
		}

		func (xi *interpolateExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		argsCopy := make([]starlarkExpr, len(xi.args))
		for i, arg := range xi.args {
		argsCopy[i] = arg.transform(transformer)
		}
		xi.args = argsCopy
		if replacement := transformer(xi); replacement != nil {
		return replacement
		} else {
		return xi
		}
		}

		type variableRefExpr struct {
		ref variable
		isDefined bool
		@@ -220,6 +263,14 @@ func (v variableRefExpr) emitListVarCopy(gctx generationContext) {
		}
		}

		func (v *variableRefExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(v); replacement != nil {
		return replacement
		} else {
		return v
		}
		}

		type toStringExpr struct {
		expr starlarkExpr
		}
		@@ -265,6 +316,15 @@ func (s toStringExpr) emitListVarCopy(gctx generationContext) {
		s.emit(gctx)
		}

		func (s *toStringExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		s.expr = s.expr.transform(transformer)
		if replacement := transformer(s); replacement != nil {
		return replacement
		} else {
		return s
		}
		}

		type notExpr struct {
		expr starlarkExpr
		}
		@@ -291,6 +351,15 @@ func (n notExpr) emitListVarCopy(gctx generationContext) {
		n.emit(gctx)
		}

		func (n *notExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		n.expr = n.expr.transform(transformer)
		if replacement := transformer(n); replacement != nil {
		return replacement
		} else {
		return n
		}
		}

		type eqExpr struct {
		left, right starlarkExpr
		isEq bool // if false, it's !=
		@@ -360,6 +429,16 @@ func (eq eqExpr) emitListVarCopy(gctx generationContext) {
		eq.emit(gctx)
		}

		func (eq *eqExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		eq.left = eq.left.transform(transformer)
		eq.right = eq.right.transform(transformer)
		if replacement := transformer(eq); replacement != nil {
		return replacement
		} else {
		return eq
		}
		}

		// variableDefinedExpr corresponds to Make's ifdef VAR
		type variableDefinedExpr struct {
		v variable
		@@ -388,6 +467,11 @@ func (v variableDefinedExpr) emitListVarCopy(gctx generationContext) {
		v.emit(gctx)
		}

		func (v *variableDefinedExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		// TODO: VariableDefinedExpr isn't really an expression?
		return v
		}

		type listExpr struct {
		items []starlarkExpr
		}
		@@ -442,6 +526,19 @@ func (l listExpr) emitListVarCopy(gctx generationContext) {
		l.emit(gctx)
		}

		func (l *listExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		itemsCopy := make([]starlarkExpr, len(l.items))
		for i, item := range l.items {
		itemsCopy[i] = item.transform(transformer)
		}
		l.items = itemsCopy
		if replacement := transformer(l); replacement != nil {
		return replacement
		} else {
		return l
		}
		}

		func newStringListExpr(items []string) *listExpr {
		v := listExpr{}
		for _, item := range items {
		@@ -505,6 +602,19 @@ func (c concatExpr) emitListVarCopy(gctx generationContext) {
		c.emit(gctx)
		}

		func (c *concatExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		itemsCopy := make([]starlarkExpr, len(c.items))
		for i, item := range c.items {
		itemsCopy[i] = item.transform(transformer)
		}
		c.items = itemsCopy
		if replacement := transformer(c); replacement != nil {
		return replacement
		} else {
		return c
		}
		}

		// inExpr generates <expr> [not] in <list>
		type inExpr struct {
		expr starlarkExpr
		@@ -543,23 +653,33 @@ func (i inExpr) emitListVarCopy(gctx generationContext) {
		i.emit(gctx)
		}

		func (i *inExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		i.expr = i.expr.transform(transformer)
		i.list = i.list.transform(transformer)
		if replacement := transformer(i); replacement != nil {
		return replacement
		} else {
		return i
		}
		}

		type indexExpr struct {
		array starlarkExpr
		index starlarkExpr
		}

		func (ix indexExpr) emit(gctx *generationContext) {
		func (ix indexExpr) emit(gctx generationContext) {
		ix.array.emit(gctx)
		gctx.write("[")
		ix.index.emit(gctx)
		gctx.write("]")
		}

		func (ix indexExpr) typ() starlarkType {
		func (ix *indexExpr) typ() starlarkType {
		return starlarkTypeString
		}

		func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
		func (ix *indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
		newArray, isSameArray := ix.array.eval(valueMap)
		newIndex, isSameIndex := ix.index.eval(valueMap)
		if same = isSameArray && isSameIndex; same {
		@@ -570,10 +690,20 @@ func (ix indexExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, sa
		return
		}

		func (ix indexExpr) emitListVarCopy(gctx *generationContext) {
		func (ix indexExpr) emitListVarCopy(gctx generationContext) {
		ix.emit(gctx)
		}

		func (ix *indexExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		ix.array = ix.array.transform(transformer)
		ix.index = ix.index.transform(transformer)
		if replacement := transformer(ix); replacement != nil {
		return replacement
		} else {
		return ix
		}
		}

		type callExpr struct {
		object starlarkExpr // nil if static call
		name string
		@@ -642,6 +772,21 @@ func (cx callExpr) emitListVarCopy(gctx generationContext) {
		cx.emit(gctx)
		}

		func (cx *callExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if cx.object != nil {
		cx.object = cx.object.transform(transformer)
		}
		argsCopy := make([]starlarkExpr, len(cx.args))
		for i, arg := range cx.args {
		argsCopy[i] = arg.transform(transformer)
		}
		if replacement := transformer(cx); replacement != nil {
		return replacement
		} else {
		return cx
		}
		}

		type ifExpr struct {
		condition starlarkExpr
		ifTrue starlarkExpr
		@@ -691,6 +836,92 @@ func (i ifExpr) emitListVarCopy(gctx generationContext) {
		i.emit(gctx)
		}

		func (i *ifExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		i.condition = i.condition.transform(transformer)
		i.ifTrue = i.ifTrue.transform(transformer)
		i.ifFalse = i.ifFalse.transform(transformer)
		if replacement := transformer(i); replacement != nil {
		return replacement
		} else {
		return i
		}
		}

		type identifierExpr struct {
		name string
		}

		func (i *identifierExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
		return i, true
		}

		func (i identifierExpr) emit(gctx generationContext) {
		gctx.write(i.name)
		}

		func (i *identifierExpr) typ() starlarkType {
		return starlarkTypeUnknown
		}

		func (i identifierExpr) emitListVarCopy(gctx generationContext) {
		i.emit(gctx)
		}

		func (i *identifierExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(i); replacement != nil {
		return replacement
		} else {
		return i
		}
		}

		type foreachExpr struct {
		varName string
		list starlarkExpr
		action starlarkExpr
		}

		func (f *foreachExpr) eval(valueMap map[string]starlarkExpr) (res starlarkExpr, same bool) {
		list, listSame := f.list.eval(valueMap)
		action, actionSame := f.action.eval(valueMap)
		same = listSame && actionSame
		if same {
		return f, same
		} else {
		return &foreachExpr{
		varName: f.varName,
		list: list,
		action: action,
		}, same
		}
		}

		func (f foreachExpr) emit(gctx generationContext) {
		gctx.write("[")
		f.action.emit(gctx)
		gctx.write(" for " + f.varName + " in ")
		f.list.emit(gctx)
		gctx.write("]")
		}

		func (f *foreachExpr) typ() starlarkType {
		return starlarkTypeList
		}

		func (f foreachExpr) emitListVarCopy(gctx generationContext) {
		f.emit(gctx)
		}

		func (f *foreachExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		f.list = f.list.transform(transformer)
		f.action = f.action.transform(transformer)
		if replacement := transformer(f); replacement != nil {
		return replacement
		} else {
		return f
		}
		}

		type badExpr struct {
		errorLocation ErrorLocation
		message string
		@@ -714,6 +945,14 @@ func (_ badExpr) emitListVarCopy(_ generationContext) {
		panic("implement me")
		}

		func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
		if replacement := transformer(b); replacement != nil {
		return replacement
		} else {
		return b
		}
		}

		func maybeConvertToStringList(expr starlarkExpr) starlarkExpr {
		if xString, ok := expr.(*stringLiteralExpr); ok {
		return newStringListExpr(strings.Fields(xString.literal))

mk2rbc/mk2rbc.go

+38 −7

Original line number	Diff line number	Diff line
		@@ -111,6 +111,7 @@ var knownFunctions = map[string]struct {
		"filter": {baseName + ".filter", starlarkTypeList, hiddenArgNone},
		"filter-out": {baseName + ".filter_out", starlarkTypeList, hiddenArgNone},
		"firstword": {"!firstword", starlarkTypeString, hiddenArgNone},
		"foreach": {"!foreach", starlarkTypeList, hiddenArgNone},
		"get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList, hiddenArgNone}, // internal macro, used by is-board-platform, etc.
		"if": {"!if", starlarkTypeUnknown, hiddenArgNone},
		"info": {baseName + ".mkinfo", starlarkTypeVoid, hiddenArgNone},
		@@ -147,14 +148,10 @@ var knownFunctions = map[string]struct {
		"warning": {baseName + ".mkwarning", starlarkTypeVoid, hiddenArgNone},
		"word": {baseName + "!word", starlarkTypeString, hiddenArgNone},
		"wildcard": {baseName + ".expand_wildcard", starlarkTypeList, hiddenArgNone},
		"words": {baseName + ".words", starlarkTypeList, hiddenArgNone},
		}

		var builtinFuncRex = regexp.MustCompile(
		"^(addprefix\|addsuffix\|abspath\|and\|basename\|call\|dir\|error\|eval" +
		"\|flavor\|foreach\|file\|filter\|filter-out\|findstring\|firstword\|guile" +
		"\|if\|info\|join\|lastword\|notdir\|or\|origin\|patsubst\|realpath" +
		"\|shell\|sort\|strip\|subst\|suffix\|value\|warning\|word\|wordlist\|words" +
		"\|wildcard)")
		var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")

		// Conversion request parameters
		type Request struct {
		@@ -1412,6 +1409,8 @@ func (ctx parseContext) parseReference(node mkparser.Node, ref mkparser.MakeSt
		switch expr.name {
		case "if":
		return ctx.parseIfFunc(node, args)
		case "foreach":
		return ctx.parseForeachFunc(node, args)
		case "word":
		return ctx.parseWordFunc(node, args)
		case "firstword", "lastword":
		@@ -1496,6 +1495,38 @@ func (ctx parseContext) parseIfFunc(node mkparser.Node, args mkparser.MakeStri
		}
		}

		func (ctx parseContext) parseForeachFunc(node mkparser.Node, args mkparser.MakeString) starlarkExpr {
		words := args.Split(",")
		if len(words) != 3 {
		return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
		}
		if !words[0].Const() \|\| words[0].Empty() \|\| !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
		return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier")
		}
		loopVarName := words[0].Strings[0]
		list := ctx.parseMakeString(node, words[1])
		action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr {
		if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
		return &identifierExpr{loopVarName}
		}
		return nil
		})

		if list.typ() != starlarkTypeList {
		list = &callExpr{
		name: "words",
		returnType: knownFunctions["words"].returnType,
		args: []starlarkExpr{list},
		}
		}

		return &foreachExpr{
		varName: loopVarName,
		list: list,
		action: action,
		}
		}

		func (ctx parseContext) parseWordFunc(node mkparser.Node, args mkparser.MakeString) starlarkExpr {
		words := args.Split(",")
		if len(words) != 2 {
		@@ -1517,7 +1548,7 @@ func (ctx parseContext) parseWordFunc(node mkparser.Node, args mkparser.MakeSt
		if array.typ() != starlarkTypeList {
		array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
		}
		return indexExpr{array, &intLiteralExpr{int(index - 1)}}
		return &indexExpr{array, &intLiteralExpr{int(index - 1)}}
		}

		func (ctx parseContext) parseFirstOrLastwordFunc(node mkparser.Node, name string, args mkparser.MakeString) starlarkExpr {

mk2rbc/mk2rbc_test.go

+22 −0

Original line number	Diff line number	Diff line
		@@ -1156,6 +1156,28 @@ def init(g, handle):
		g["SOURCES"] = "foo.c bar.c"
		g["OBJECTS"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
		g["OBJECTS2"] = rblf.mkpatsubst("%.c", "%.o", g["SOURCES"])
		`,
		},
		{
		desc: "foreach expressions",
		mkname: "product.mk",
		in: `
		BOOT_KERNEL_MODULES := foo.ko bar.ko
		BOOT_KERNEL_MODULES_FILTER := $(foreach m,$(BOOT_KERNEL_MODULES),%/$(m))
		BOOT_KERNEL_MODULES_LIST := foo.ko
		BOOT_KERNEL_MODULES_LIST += bar.ko
		BOOT_KERNEL_MODULES_FILTER_2 := $(foreach m,$(BOOT_KERNEL_MODULES_LIST),%/$(m))

		`,
		expected: `load("//build/make/core:product_config.rbc", "rblf")

		def init(g, handle):
		cfg = rblf.cfg(handle)
		g["BOOT_KERNEL_MODULES"] = "foo.ko bar.ko"
		g["BOOT_KERNEL_MODULES_FILTER"] = ["%%/%s" % m for m in rblf.words(g["BOOT_KERNEL_MODULES"])]
		g["BOOT_KERNEL_MODULES_LIST"] = ["foo.ko"]
		g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"]
		g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]]
		`,
		},
		}