Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fa739da0 authored by Bob Badour's avatar Bob Badour
Browse files

compliance package for license metadata: dumpgraph 

package to read, consume, and analyze license metadata and dependency
graph.

Includes testdata/ and the the below command-line tool:

dumpgraph outputs edges of the graph as "target dependency annotations"

Bug: 68860345
Bug: 151177513
Bug: 151953481

Test: m all
Test: m systemlicense
Test: m dumpgraph; out/soong/host/linux-x86/dumpgraph ...

where ... is the path to the .meta_lic file for the system image. In my
case if

$ export PRODUCT=$(realpath $ANDROID_PRODUCT_OUT --relative-to=$PWD)

... can be expressed as:

${PRODUCT}/gen/META/lic_intermediates/${PRODUCT}/system.img.meta_lic

Change-Id: I5fe57d361da5155dbcb2c0d369626e9200c9d664
parent 8fb2d834

tools/compliance/Android.bp

+7 −0
Original line number Original line Diff line number Diff line
@@ -17,6 +17,13 @@ package { 
    default_applicable_licenses: ["Android-Apache-2.0"],

    default_applicable_licenses: ["Android-Apache-2.0"],

}

}





blueprint_go_binary {

    name: "dumpgraph",

    srcs: ["cmd/dumpgraph.go"],

    deps: ["compliance-module"],

    testSrcs: ["cmd/dumpgraph_test.go"],

}



bootstrap_go_package {

bootstrap_go_package {

    name: "compliance-module",

    name: "compliance-module",

    srcs: [

    srcs: [

tools/compliance/cmd/dumpgraph.go

0 → 100644
+178 −0
Original line number Original line Diff line number Diff line 
// Copyright 2021 Google LLC

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

//      http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.



package main



import (

	"compliance"

	"flag"

	"fmt"

	"io"

	"os"

	"path/filepath"

	"sort"

	"strings"

)



var (

	graphViz        = flag.Bool("dot", false, "Whether to output graphviz (i.e. dot) format.")

	labelConditions = flag.Bool("label_conditions", false, "Whether to label target nodes with conditions.")

	stripPrefix     = flag.String("strip_prefix", "", "Prefix to remove from paths. i.e. path to root")



	failNoneRequested = fmt.Errorf("\nNo license metadata files requested")

	failNoLicenses = fmt.Errorf("No licenses found")

)



type context struct {

	graphViz        bool

	labelConditions bool

	stripPrefix     string

}



func init() {

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...}



Outputs space-separated Target Dependency Annotations tuples for each

edge in the license graph. When -dot flag given, outputs the nodes and

edges in graphViz directed graph format.



In plain text mode, multiple values within a field are colon-separated.

e.g. multiple annotations appear as annotation1:annotation2:annotation3

or when -label_conditions is requested, Target and Dependency become

target:condition1:condition2 etc.



Options:

`, filepath.Base(os.Args[0]))

		flag.PrintDefaults()

	}

}



func main() {

	flag.Parse()



	// Must specify at least one root target.

	if flag.NArg() == 0 {

		flag.Usage()

		os.Exit(2)

	}



	ctx := &context{*graphViz, *labelConditions, *stripPrefix}



	err := dumpGraph(ctx, os.Stdout, os.Stderr, flag.Args()...)

	if err != nil {

		if err == failNoneRequested {

			flag.Usage()

		}

		fmt.Fprintf(os.Stderr, "%s\n", err.Error())

		os.Exit(1)

	}

	os.Exit(0)

}



// dumpGraph implements the dumpgraph utility.

func dumpGraph(ctx *context, stdout, stderr io.Writer, files ...string) error {

	if len(files) < 1 {

		return failNoneRequested

	}



	// Read the license graph from the license metadata files (*.meta_lic).

	licenseGraph, err := compliance.ReadLicenseGraph(os.DirFS("."), stderr, files)

	if err != nil {

		return fmt.Errorf("Unable to read license metadata file(s) %q: %w\n", files, err)

	}

	if licenseGraph == nil {

		return failNoLicenses

	}



	// Sort the edges of the graph.

	edges := licenseGraph.Edges()

	sort.Sort(edges)



	// nodes maps license metadata file names to graphViz node names when ctx.graphViz is true.

	var nodes map[string]string

	n := 0



	// targetOut calculates the string to output for `target` separating conditions as needed using `sep`.

	targetOut := func(target *compliance.TargetNode, sep string) string {

		tOut := strings.TrimPrefix(target.Name(), ctx.stripPrefix)

		if ctx.labelConditions {

			conditions := target.LicenseConditions().Names()

			sort.Strings(conditions)

			if len(conditions) > 0 {

				tOut += sep + strings.Join(conditions, sep)

			}

		}

		return tOut

	}



	// makeNode maps `target` to a graphViz node name.

	makeNode := func(target *compliance.TargetNode) {

		tName := target.Name()

		if _, ok := nodes[tName]; !ok {

			nodeName := fmt.Sprintf("n%d", n)

			nodes[tName] = nodeName

			fmt.Fprintf(stdout, "\t%s [label=\"%s\"];\n", nodeName, targetOut(target, "\\n"))

			n++

		}

	}



	// If graphviz output, map targets to node names, and start the directed graph.

	if ctx.graphViz {

		nodes = make(map[string]string)

		targets := licenseGraph.Targets()

		sort.Sort(targets)



		fmt.Fprintf(stdout, "strict digraph {\n\trankdir=RL;\n")

		for _, target := range targets {

			makeNode(target)

		}

	}



	// Print the sorted edges to stdout ...

	for _, e := range edges {

		// sort the annotations for repeatability/stability

		annotations := e.Annotations().AsList()

		sort.Strings(annotations)



		tName := e.Target().Name()

		dName := e.Dependency().Name()



		if ctx.graphViz {

			// ... one edge per line labelled with \\n-separated annotations.

			tNode := nodes[tName]

			dNode := nodes[dName]

			fmt.Fprintf(stdout, "\t%s -> %s [label=\"%s\"];\n", dNode, tNode, strings.Join(annotations, "\\n"))

		} else {

			// ... one edge per line with annotations in a colon-separated tuple.

			fmt.Fprintf(stdout, "%s %s %s\n", targetOut(e.Target(), ":"), targetOut(e.Dependency(), ":"), strings.Join(annotations, ":"))

		}

	}



	// If graphViz output, rank the root nodes together, and complete the directed graph.

	if ctx.graphViz {

		fmt.Fprintf(stdout, "\t{rank=same;")

		for _, f := range files {

			fName := f

			if !strings.HasSuffix(fName, ".meta_lic") {

				fName += ".meta_lic"

			}

			if fNode, ok := nodes[fName]; ok {

				fmt.Fprintf(stdout, " %s", fNode)

			}

		}

		fmt.Fprintf(stdout, "}\n}\n")

	}

	return nil

}

tools/compliance/cmd/dumpgraph_test.go

0 → 100644
+1258 −0

File added.

Preview size limit exceeded, changes collapsed.

tools/compliance/cmd/testdata/README.md

0 → 100644
+321 −0
Original line number Original line Diff line number Diff line 
## Test data



Each directory under testdata/ defines a similar build graph.

All have the same structure, but different versions of the graph have different

license metadata.



### Testdata build graph structure:



The structure is meant to simulate some common scenarios:



*   a `lib/` directory with some libraries

*   a `bin/` directory with some executables

*   one of the binaries, `bin3`, is a toolchain executable like a compiler

*   an `application` built with the `bin3` compiler and linking a couple libraries

*   a pure aggregation `continer.zip` that merely bundles files together, and

*   an apex file (more like an apk file) with some binaries and libraries.



The testdata starts with a `firstparty/` version containng only first-party

licenses, and each subsequent directory introduces more restrictive conditions:



*   `notice/` starts with `firstparty/` adds third-party notice conditions

*   `reciprocal/` starts with `notice/` and adds some reciprocal conditions

*   `restricted/` starts with `reciprocal/` and adds some restricted conditions

*   `proprietary/` starts with `restricted/` and add some privacy conditions



#### a `lib/` directory with some libraries



```dot

strict digraph {

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	libc [label="lib/libc.a.meta_lic"];

	libd [label="lib/libd.so.meta_lic"];

}

```



#### a `bin/` directory with some executables



strict digraph {

	rankdir=LR;

	bin1 [label="bin/bin1.meta_lic"];

	bin2 [label="bin/bin2.meta_lic"];

	bin3 [label="bin/bin3.meta_lic\ntoolchain"];

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	libc [label="lib/libc.a.meta_lic"];

	libd [label="lib/libd.so.meta_lic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	{rank=same; bin1 bin2 bin3}

}



#### an `application` built with the `bin3` compiler and linking a couple libraries



```dot

strict digraph {

	rankdir=LR;

	app [label="application.meta_lic"];

	bin3 [label="bin/bin3.meta_lic"];

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	{rank=same; app}

}

```



#### a pure aggregation `container.zip` that merely bundles files together



strict digraph {

	rankdir=LR;

	bin1 [label="bin/bin1.meta_lic"];

	bin2 [label="bin/bin2.meta_lic"];

	container [label="container.zip.meta_lic"];

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	libc [label="lib/libc.a.meta_lic"];

	libd [label="lib/libd.so.meta_lic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	{rank=same; container}

}



#### an apex file (more like an apk file) with some binaries and libraries



```dot

strict digraph {

	rankdir=LR;

	apex [label="highest.apex.meta_lic"];

	bin1 [label="bin/bin1.meta_lic"];

	bin2 [label="bin/bin2.meta_lic"];

	bin3 [label="bin/bin3.meta_lic"];

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	libc [label="lib/libc.a.meta_lic"];

	libd [label="lib/libd.so.meta_lic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; apex}

}

```



#### the whole build graph



```dot

strict digraph {

	rankdir=LR;

	apex [label="highest.apex.meta_lic"];

	app [label="application.meta_lic"];

	bin1 [label="bin/bin1.meta_lic"];

	bin2 [label="bin/bin2.meta_lic"];

	bin3 [label="bin/bin3.meta_lic"];

	container [label="container.zip.meta_lic"];

	liba [label="lib/liba.so.meta_lic"];

	libb [label="lib/libb.so.meta_lic"];

	libc [label="lib/libc.a.meta_lic"];

	libd [label="lib/libd.so.meta_lic"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```





### firstparty/ testdata starts with all first-party licensing



```dot

strict digraph {

	rankdir=LR;

	app [label="firstparty/application.meta_lic"];

	bin1 [label="firstparty/bin/bin1.meta_lic"];

	bin2 [label="firstparty/bin/bin2.meta_lic"];

	bin3 [label="firstparty/bin/bin3.meta_lic"];

	container [label="firstparty/container.zip.meta_lic"];

	apex [label="firstparty/highest.apex.meta_lic"];

	liba [label="firstparty/lib/liba.so.meta_lic"];

	libb [label="firstparty/lib/libb.so.meta_lic"];

	libc [label="firstparty/lib/libc.a.meta_lic"];

	lib [label="firstparty/lib/libd.so.meta_lic"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```



### notice/ testdata introduces third-party notice conditions



```dot

strict digraph {

	rankdir=LR;

	app [label="notice/application.meta_lic"];

	bin1 [label="notice/bin/bin1.meta_lic"];

	bin2 [label="notice/bin/bin2.meta_lic"];

	bin3 [label="notice/bin/bin3.meta_lic\nnotice"];

	container [label="notice/container.zip.meta_lic"];

	apex [label="notice/highest.apex.meta_lic"];

	liba [label="notice/lib/liba.so.meta_lic\nnotice"];

	libb [label="notice/lib/libb.so.meta_lic"];

	libc [label="notice/lib/libc.a.meta_lic\nnotice"];

	libd [label="notice/lib/libd.so.meta_lic\nnotice"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```



### reciprocal/ testdata introduces third-party reciprocal sharing conditions



```dot

strict digraph {

	rankdir=LR;

	app [label="reciprocal/application.meta_lic"];

	bin1 [label="reciprocal/bin/bin1.meta_lic"];

	bin2 [label="reciprocal/bin/bin2.meta_lic"];

	bin3 [label="reciprocal/bin/bin3.meta_lic\nnotice"];

	container [label="reciprocal/container.zip.meta_lic"];

	apex [label="reciprocal/highest.apex.meta_lic"];

	liba [label="reciprocal/lib/liba.so.meta_lic\nreciprocal"];

	libb [label="reciprocal/lib/libb.so.meta_lic"];

	libc [label="reciprocal/lib/libc.a.meta_lic\nreciprocal"];

	libd [label="reciprocal/lib/libd.so.meta_lic\nnotice"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```



### restricted/ testdata introduces restricted source sharing conditions



```dot

strict digraph {

	rankdir=LR;

	app [label="restricted/application.meta_lic"];

	bin1 [label="restricted/bin/bin1.meta_lic"];

	bin2 [label="restricted/bin/bin2.meta_lic"];

	bin3 [label="restricted/bin/bin3.meta_lic\nrestricted"];

	container [label="restricted/container.zip.meta_lic"];

	apex [label="restricted/highest.apex.meta_lic"];

	liba [label="restricted/lib/liba.so.meta_lic\nrestricted"];

	libb [label="restricted/lib/libb.so.meta_lic\nrestricted"];

	libc [label="restricted/lib/libc.a.meta_lic\nreciprocal"];

	libd [label="restricted/lib/libd.so.meta_lic\nnotice"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```



### proprietary/ testdata introduces privacy conditions



```dot

strict digraph {

	rankdir=LR;

	app [label="proprietary/application.meta_lic"];

	bin1 [label="proprietary/bin/bin1.meta_lic"];

	bin2 [label="proprietary/bin/bin2.meta_lic\nby_exception_only\nproprietary"];

	bin3 [label="proprietary/bin/bin3.meta_lic\nrestricted"];

	container [label="proprietary/container.zip.meta_lic"];

	apex [label="proprietary/highest.apex.meta_lic"];

	liba [label="proprietary/lib/liba.so.meta_lic\nby_exception_only\nproprietary"];

	libb [label="proprietary/lib/libb.so.meta_lic\nrestricted"];

	libc [label="proprietary/lib/libc.a.meta_lic\nby_exception_only\nproprietary"];

	libd [label="proprietary/lib/libd.so.meta_lic\nnotice"];

	app -> bin3 [label="toolchain"];

	app -> liba [label="static"];

	app -> libb [label="dynamic"];

	bin1 -> liba [label="static"];

	bin1 -> libc [label="static"];

	bin2 -> libb [label="dynamic"];

	bin2 -> libd [label="dynamic"];

	container -> bin1 [label="static"];

	container -> bin2 [label="static"];

	container -> liba [label="static"];

	container -> libb [label="static"];

	apex -> bin1 [label="static"];

	apex -> bin2 [label="static"];

	apex -> liba [label="static"];

	apex -> libb [label="static"];

	{rank=same; app container apex}

}

```

tools/compliance/cmd/testdata/firstparty/application.meta_lic

0 → 100644
+24 −0
Original line number Original line Diff line number Diff line 
package_name:  "Android"

module_classes: "EXECUTABLES"

projects:  "distributable/application"

license_kinds:  "SPDX-license-identifier-Apache-2.0"

license_conditions:  "notice"

license_texts:  "build/soong/licenses/LICENSE"

is_container:  false

built:  "out/target/product/fictional/obj/EXECUTABLES/application_intermediates/application"

installed:  "out/target/product/fictional/bin/application"

sources:  "out/target/product/fictional/system/lib/liba.a"

sources:  "out/target/product/fictional/system/lib/libb.so"

sources:  "out/target/product/fictional/system/bin/bin3"

deps:  {

  file:  "testdata/firstparty/bin/bin3.meta_lic"

  annotations:  "toolchain"

}

deps:  {

  file:  "testdata/firstparty/lib/liba.so.meta_lic"

  annotations:  "static"

}

deps:  {

  file:  "testdata/firstparty/lib/libb.so.meta_lic"

  annotations:  "dynamic"

}