Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 103eb0f9 authored by Bob Badour's avatar Bob Badour
Browse files

Performance and scale.

Defer edge creation.

Don't create edges until the count is known to avoid repeated allocate+
copy operatios.

Limit resolutions.

Allow only a single resolution condition set per target, and overwrite
intermediate results. Reduces memory and obviates allocations.

Propagate fewer conditions.

Instead of propagating notice conditions to parents in graph during
initial resolve, leave them on leaf node, and attach to ancestors in
the final walk. Reduces copies.

Parallelize resolutions.

Use goroutines, mutexes, and waitgroups to resolve branches of the
graph in parallel. Makes better use of available cores.

Don't accumulate resolutions inside non-containers.

During the final resolution walk, only attach actions to ancestors from
the root down until the 1st non-aggregate. Prevents an explosion of
copies in the lower levels of the graph.

Drop origin for scale.

Tracking the origin of every potential origin for every restricted
condition does not scale. By dropping origin, propagating from top
to bottom can prune many redundant paths avoiding an exponential
explosion.

Conditions as bitmask.

Use bit masks for license conditions and condition sets. Reduces maps
and allocations.

Bug: 68860345
Bug: 151177513
Bug: 151953481

Test: m all
Test: m systemlicense
Test: m listshare; out/soong/host/linux-x86/bin/listshare ...
Test: m checkshare; out/soong/host/linux-x86/bin/checkshare ...
Test: m dumpgraph; out/soong/host/linux-x86/dumpgraph ...
Test: m dumpresolutions; out/soong/host/linux-x86/dumpresolutions ...

where ... is the path to the .meta_lic file for the system image. In my
case if

$ export PRODUCT=$(realpath $ANDROID_PRODUCT_OUT --relative-to=$PWD)

... can be expressed as:

${PRODUCT}/gen/META/lic_intermediates/${PRODUCT}/system.img.meta_lic

Change-Id: Ia2ec1b818de6122c239fbd0824754f1d65daffd3
parent 5446a6f8
Loading
Loading
Loading
Loading
+0 −1
Original line number Diff line number Diff line
@@ -48,7 +48,6 @@ blueprint_go_binary {
bootstrap_go_package {
    name: "compliance-module",
    srcs: [
        "actionset.go",
        "condition.go",
        "conditionset.go",
        "doc.go",

tools/compliance/actionset.go

deleted100644 → 0
+0 −119
Original line number Diff line number Diff line
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compliance

import (
	"fmt"
	"sort"
	"strings"
)

// actionSet maps `actOn` target nodes to the license conditions the actions resolve.
type actionSet map[*TargetNode]*LicenseConditionSet

// String returns a string representation of the set.
func (as actionSet) String() string {
	var sb strings.Builder
	fmt.Fprintf(&sb, "{")
	osep := ""
	for actsOn, cs := range as {
		cl := cs.AsList()
		sort.Sort(cl)
		fmt.Fprintf(&sb, "%s%s -> %s", osep, actsOn.name, cl.String())
		osep = ", "
	}
	fmt.Fprintf(&sb, "}")
	return sb.String()
}

// byName returns the subset of `as` actions where the condition name is in `names`.
func (as actionSet) byName(names ConditionNames) actionSet {
	result := make(actionSet)
	for actsOn, cs := range as {
		bn := cs.ByName(names)
		if bn.IsEmpty() {
			continue
		}
		result[actsOn] = bn
	}
	return result
}

// byActsOn returns the subset of `as` where `actsOn` is in the `reachable` target node set.
func (as actionSet) byActsOn(reachable *TargetNodeSet) actionSet {
	result := make(actionSet)
	for actsOn, cs := range as {
		if !reachable.Contains(actsOn) || cs.IsEmpty() {
			continue
		}
		result[actsOn] = cs.Copy()
	}
	return result
}

// copy returns another actionSet with the same value as `as`
func (as actionSet) copy() actionSet {
	result := make(actionSet)
	for actsOn, cs := range as {
		if cs.IsEmpty() {
			continue
		}
		result[actsOn] = cs.Copy()
	}
	return result
}

// addSet adds all of the actions of `other` if not already present.
func (as actionSet) addSet(other actionSet) {
	for actsOn, cs := range other {
		as.add(actsOn, cs)
	}
}

// add makes the action on `actsOn` to resolve the conditions in `cs` a member of the set.
func (as actionSet) add(actsOn *TargetNode, cs *LicenseConditionSet) {
	if acs, ok := as[actsOn]; ok {
		acs.AddSet(cs)
	} else {
		as[actsOn] = cs.Copy()
	}
}

// addCondition makes the action on `actsOn` to resolve `lc` a member of the set.
func (as actionSet) addCondition(actsOn *TargetNode, lc LicenseCondition) {
	if _, ok := as[actsOn]; !ok {
		as[actsOn] = newLicenseConditionSet()
	}
	as[actsOn].Add(lc)
}

// isEmpty returns true if no action to resolve a condition exists.
func (as actionSet) isEmpty() bool {
	for _, cs := range as {
		if !cs.IsEmpty() {
			return false
		}
	}
	return true
}

// conditions returns the set of conditions resolved by the action set.
func (as actionSet) conditions() *LicenseConditionSet {
	result := newLicenseConditionSet()
	for _, cs := range as {
		result.AddSet(cs)
	}
	return result
}
+2 −2
Original line number Diff line number Diff line
@@ -30,8 +30,8 @@ func init() {

Reports on stderr any targets where policy says that the source both
must and must not be shared. The error report indicates the target, the
license condition with origin that has a source privacy policy, and the
license condition with origin that has a source sharing policy.
license condition that has a source privacy policy, and the license
condition that has a source sharing policy.

Any given target may appear multiple times with different combinations
of conflicting license conditions.
+2 −15
Original line number Diff line number Diff line
@@ -23,15 +23,12 @@ import (

type outcome struct {
	target           string
	privacyOrigin    string
	privacyCondition string
	shareOrigin      string
	shareCondition   string
}

func (o *outcome) String() string {
	return fmt.Sprintf("%s %s from %s and must share from %s %s",
		o.target, o.privacyCondition, o.privacyOrigin, o.shareCondition, o.shareOrigin)
	return fmt.Sprintf("%s %s and must share from %s", o.target, o.privacyCondition, o.shareCondition)
}

type outcomeList []*outcome
@@ -180,9 +177,7 @@ func Test(t *testing.T) {
			expectedOutcomes: outcomeList{
				&outcome{
					target:           "testdata/proprietary/bin/bin2.meta_lic",
					privacyOrigin:    "testdata/proprietary/bin/bin2.meta_lic",
					privacyCondition: "proprietary",
					shareOrigin:      "testdata/proprietary/lib/libb.so.meta_lic",
					shareCondition:   "restricted",
				},
			},
@@ -195,9 +190,7 @@ func Test(t *testing.T) {
			expectedOutcomes: outcomeList{
				&outcome{
					target:           "testdata/proprietary/bin/bin2.meta_lic",
					privacyOrigin:    "testdata/proprietary/bin/bin2.meta_lic",
					privacyCondition: "proprietary",
					shareOrigin:      "testdata/proprietary/lib/libb.so.meta_lic",
					shareCondition:   "restricted",
				},
			},
@@ -210,9 +203,7 @@ func Test(t *testing.T) {
			expectedOutcomes: outcomeList{
				&outcome{
					target:           "testdata/proprietary/lib/liba.so.meta_lic",
					privacyOrigin:    "testdata/proprietary/lib/liba.so.meta_lic",
					privacyCondition: "proprietary",
					shareOrigin:      "testdata/proprietary/lib/libb.so.meta_lic",
					shareCondition:   "restricted",
				},
			},
@@ -225,9 +216,7 @@ func Test(t *testing.T) {
			expectedOutcomes: outcomeList{
				&outcome{
					target:           "testdata/proprietary/bin/bin2.meta_lic",
					privacyOrigin:    "testdata/proprietary/bin/bin2.meta_lic",
					privacyCondition: "proprietary",
					shareOrigin:      "testdata/proprietary/lib/libb.so.meta_lic",
					shareCondition:   "restricted",
				},
			},
@@ -277,10 +266,8 @@ func Test(t *testing.T) {
				cFields := strings.Split(ts, " ")
				actualOutcomes = append(actualOutcomes, &outcome{
					target:           cFields[0],
					privacyOrigin:    cFields[3],
					privacyCondition: cFields[1],
					shareOrigin:      cFields[9],
					shareCondition:   cFields[8],
					shareCondition:   cFields[6],
				})
			}
			if len(actualOutcomes) != len(tt.expectedOutcomes) {
+3 −3
Original line number Diff line number Diff line
@@ -327,13 +327,13 @@ func Test_plaintext(t *testing.T) {
			roots:     []string{"highest.apex.meta_lic"},
			ctx:       context{stripPrefix: "testdata/restricted/", labelConditions: true},
			expectedOut: []string{
				"bin/bin1.meta_lic:notice lib/liba.so.meta_lic:restricted static",
				"bin/bin1.meta_lic:notice lib/liba.so.meta_lic:restricted_allows_dynamic_linking static",
				"bin/bin1.meta_lic:notice lib/libc.a.meta_lic:reciprocal static",
				"bin/bin2.meta_lic:notice lib/libb.so.meta_lic:restricted dynamic",
				"bin/bin2.meta_lic:notice lib/libd.so.meta_lic:notice dynamic",
				"highest.apex.meta_lic:notice bin/bin1.meta_lic:notice static",
				"highest.apex.meta_lic:notice bin/bin2.meta_lic:notice static",
				"highest.apex.meta_lic:notice lib/liba.so.meta_lic:restricted static",
				"highest.apex.meta_lic:notice lib/liba.so.meta_lic:restricted_allows_dynamic_linking static",
				"highest.apex.meta_lic:notice lib/libb.so.meta_lic:restricted static",
			},
		},
@@ -996,7 +996,7 @@ func Test_graphviz(t *testing.T) {
				matchTarget("bin/bin1.meta_lic", "notice"),
				matchTarget("bin/bin2.meta_lic", "notice"),
				matchTarget("highest.apex.meta_lic", "notice"),
				matchTarget("lib/liba.so.meta_lic", "restricted"),
				matchTarget("lib/liba.so.meta_lic", "restricted_allows_dynamic_linking"),
				matchTarget("lib/libb.so.meta_lic", "restricted"),
				matchTarget("lib/libc.a.meta_lic", "reciprocal"),
				matchTarget("lib/libd.so.meta_lic", "notice"),
Loading