Loading tools/compliance/Android.bp +10 −0 Original line number Diff line number Diff line Loading @@ -86,6 +86,16 @@ blueprint_go_binary { testSrcs: ["cmd/textnotice/textnotice_test.go"], } blueprint_go_binary { name: "xmlnotice", srcs: ["cmd/xmlnotice/xmlnotice.go"], deps: [ "compliance-module", "blueprint-deptools", ], testSrcs: ["cmd/xmlnotice/xmlnotice_test.go"], } bootstrap_go_package { name: "compliance-module", srcs: [ Loading tools/compliance/cmd/xmlnotice/xmlnotice.go 0 → 100644 +197 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "compress/gzip" "encoding/xml" "flag" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "android/soong/tools/compliance" "github.com/google/blueprint/deptools" ) var ( outputFile = flag.String("o", "-", "Where to write the NOTICE xml or xml.gz file. (default stdout)") depsFile = flag.String("d", "", "Where to write the deps file") stripPrefix = flag.String("strip_prefix", "", "Prefix to remove from paths. i.e. path to root") failNoneRequested = fmt.Errorf("\nNo license metadata files requested") failNoLicenses = fmt.Errorf("No licenses found") ) type context struct { stdout io.Writer stderr io.Writer rootFS fs.FS stripPrefix string deps *[]string } func init() { flag.Usage = func() { fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} Outputs an xml NOTICE.xml or gzipped NOTICE.xml.gz file if the -o filename ends with ".gz". Options: `, filepath.Base(os.Args[0])) flag.PrintDefaults() } } func main() { flag.Parse() // Must specify at least one root target. if flag.NArg() == 0 { flag.Usage() os.Exit(2) } if len(*outputFile) == 0 { flag.Usage() fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") os.Exit(2) } else { dir, err := filepath.Abs(filepath.Dir(*outputFile)) if err != nil { fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err) os.Exit(1) } fi, err := os.Stat(dir) if err != nil { fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err) os.Exit(1) } if !fi.IsDir() { fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) os.Exit(1) } } var ofile io.Writer var closer io.Closer ofile = os.Stdout var obuf *bytes.Buffer if *outputFile != "-" { obuf = &bytes.Buffer{} ofile = obuf } if strings.HasSuffix(*outputFile, ".gz") { ofile, _ = gzip.NewWriterLevel(obuf, gzip.BestCompression) closer = ofile.(io.Closer) } var deps []string ctx := &context{ofile, os.Stderr, os.DirFS("."), *stripPrefix, &deps} err := xmlNotice(ctx, flag.Args()...) if err != nil { if err == failNoneRequested { flag.Usage() } fmt.Fprintf(os.Stderr, "%s\n", err.Error()) os.Exit(1) } if closer != nil { closer.Close() } if *outputFile != "-" { err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) if err != nil { fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err) os.Exit(1) } } if *depsFile != "" { err := deptools.WriteDepFile(*depsFile, *outputFile, deps) if err != nil { fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err) os.Exit(1) } } os.Exit(0) } // xmlNotice implements the xmlnotice utility. func xmlNotice(ctx *context, files ...string) error { // Must be at least one root file. if len(files) < 1 { return failNoneRequested } // Read the license graph from the license metadata files (*.meta_lic). licenseGraph, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) if err != nil { return fmt.Errorf("Unable to read license metadata file(s) %q: %v\n", files, err) } if licenseGraph == nil { return failNoLicenses } // rs contains all notice resolutions. rs := compliance.ResolveNotices(licenseGraph) ni, err := compliance.IndexLicenseTexts(ctx.rootFS, licenseGraph, rs) if err != nil { return fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) } fmt.Fprintln(ctx.stdout, "<?xml version=\"1.0\" encoding=\"utf-8\"?>") fmt.Fprintln(ctx.stdout, "<licenses>") for installPath := range ni.InstallPaths() { var p string if 0 < len(ctx.stripPrefix) && strings.HasPrefix(installPath, ctx.stripPrefix) { p = installPath[len(ctx.stripPrefix):] if 0 == len(p) { p = "root" } } else { p = installPath } for _, h := range ni.InstallHashes(installPath) { for _, lib := range ni.InstallHashLibs(installPath, h) { fmt.Fprintf(ctx.stdout, "<file-name contentId=\"%s\" lib=\"", h.String()) xml.EscapeText(ctx.stdout, []byte(lib)) fmt.Fprintf(ctx.stdout, "\">") xml.EscapeText(ctx.stdout, []byte(p)) fmt.Fprintln(ctx.stdout, "</file-name>") } } } for h := range ni.Hashes() { fmt.Fprintf(ctx.stdout, "<file-content contentId=\"%s\"><![CDATA[", h) xml.EscapeText(ctx.stdout, ni.HashText(h)) fmt.Fprintf(ctx.stdout, "]]></file-content>\n\n") } fmt.Fprintln(ctx.stdout, "</licenses>") *ctx.deps = ni.InputNoticeFiles() return nil } tools/compliance/cmd/xmlnotice/xmlnotice_test.go 0 → 100644 +634 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bufio" "bytes" "encoding/xml" "fmt" "os" "reflect" "regexp" "strings" "testing" ) var ( installTarget = regexp.MustCompile(`^<file-name contentId="[^"]{32}" lib="([^"]*)">([^<]+)</file-name>`) licenseText = regexp.MustCompile(`^<file-content contentId="[^"]{32}"><![[]CDATA[[]([^]]*)[]][]]></file-content>`) ) func TestMain(m *testing.M) { // Change into the parent directory before running the tests // so they can find the testdata directory. if err := os.Chdir(".."); err != nil { fmt.Printf("failed to change to testdata directory: %s\n", err) os.Exit(1) } os.Exit(m.Run()) } func Test(t *testing.T) { tests := []struct { condition string name string roots []string stripPrefix string expectedOut []matcher expectedDeps []string }{ { condition: "firstparty", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Android"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Android"}, target{"container.zip/libb.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "notice", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "reciprocal", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "restricted", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, restricted{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "proprietary", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, restricted{}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "proprietary", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, restricted{}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "proprietary", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", }, }, { condition: "proprietary", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", }, }, { condition: "proprietary", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, } for _, tt := range tests { t.Run(tt.condition+" "+tt.name, func(t *testing.T) { stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} rootFiles := make([]string, 0, len(tt.roots)) for _, r := range tt.roots { rootFiles = append(rootFiles, "testdata/"+tt.condition+"/"+r) } var deps []string ctx := context{stdout, stderr, os.DirFS("."), tt.stripPrefix, &deps} err := xmlNotice(&ctx, rootFiles...) if err != nil { t.Fatalf("xmlnotice: error = %v, stderr = %v", err, stderr) return } if stderr.Len() > 0 { t.Errorf("xmlnotice: gotStderr = %v, want none", stderr) } t.Logf("got stdout: %s", stdout.String()) t.Logf("want stdout: %s", matcherList(tt.expectedOut).String()) out := bufio.NewScanner(stdout) lineno := 0 inBody := false outOfBody := true for out.Scan() { line := out.Text() if strings.TrimLeft(line, " ") == "" { continue } if lineno == 0 && !inBody && `<?xml version="1.0" encoding="utf-8"?>` == line { continue } if !inBody { if "<licenses>" == line { inBody = true outOfBody = false } continue } else if "</licenses>" == line { outOfBody = true continue } if len(tt.expectedOut) <= lineno { t.Errorf("xmlnotice: unexpected output at line %d: got %q, want nothing (wanted %d lines)", lineno+1, line, len(tt.expectedOut)) } else if !tt.expectedOut[lineno].isMatch(line) { t.Errorf("xmlnotice: unexpected output at line %d: got %q, want %q", lineno+1, line, tt.expectedOut[lineno].String()) } lineno++ } if !inBody { t.Errorf("xmlnotice: missing <licenses> tag: got no <licenses> tag, want <licenses> tag on 2nd line") } if !outOfBody { t.Errorf("xmlnotice: missing </licenses> tag: got no </licenses> tag, want </licenses> tag on last line") } for ; lineno < len(tt.expectedOut); lineno++ { t.Errorf("xmlnotice: missing output line %d: ended early, want %q", lineno+1, tt.expectedOut[lineno].String()) } t.Logf("got deps: %q", deps) t.Logf("want deps: %q", tt.expectedDeps) if g, w := deps, tt.expectedDeps; !reflect.DeepEqual(g, w) { t.Errorf("unexpected deps, wanted:\n%s\ngot:\n%s\n", strings.Join(w, "\n"), strings.Join(g, "\n")) } }) } } func escape(s string) string { b := &bytes.Buffer{} xml.EscapeText(b, []byte(s)) return b.String() } type matcher interface { isMatch(line string) bool String() string } type target struct { name string lib string } func (m target) isMatch(line string) bool { groups := installTarget.FindStringSubmatch(line) if len(groups) != 3 { return false } return groups[1] == escape(m.lib) && strings.HasPrefix(groups[2], "out/") && strings.HasSuffix(groups[2], "/"+escape(m.name)) } func (m target) String() string { return `<file-name contentId="hash" lib="` + escape(m.lib) + `">` + escape(m.name) + `</file-name>` } func matchesText(line, text string) bool { groups := licenseText.FindStringSubmatch(line) if len(groups) != 2 { return false } return groups[1] == escape(text + "\n") } func expectedText(text string) string { return `<file-content contentId="hash"><![CDATA[` + escape(text + "\n") + `]]></file-content>` } type firstParty struct{} func (m firstParty) isMatch(line string) bool { return matchesText(line, "&&&First Party License&&&") } func (m firstParty) String() string { return expectedText("&&&First Party License&&&") } type notice struct{} func (m notice) isMatch(line string) bool { return matchesText(line, "%%%Notice License%%%") } func (m notice) String() string { return expectedText("%%%Notice License%%%") } type reciprocal struct{} func (m reciprocal) isMatch(line string) bool { return matchesText(line, "$$$Reciprocal License$$$") } func (m reciprocal) String() string { return expectedText("$$$Reciprocal License$$$") } type restricted struct{} func (m restricted) isMatch(line string) bool { return matchesText(line, "###Restricted License###") } func (m restricted) String() string { return expectedText("###Restricted License###") } type proprietary struct{} func (m proprietary) isMatch(line string) bool { return matchesText(line, "@@@Proprietary License@@@") } func (m proprietary) String() string { return expectedText("@@@Proprietary License@@@") } type matcherList []matcher func (l matcherList) String() string { var sb strings.Builder fmt.Fprintln(&sb, `<?xml version="1.0" encoding="utf-8"?>`) fmt.Fprintln(&sb, `<licenses>`) for _, m := range l { s := m.String() fmt.Fprintln(&sb, s) if _, ok := m.(target); !ok { fmt.Fprintln(&sb) } } fmt.Fprintln(&sb, `/<licenses>`) return sb.String() } Loading
tools/compliance/Android.bp +10 −0 Original line number Diff line number Diff line Loading @@ -86,6 +86,16 @@ blueprint_go_binary { testSrcs: ["cmd/textnotice/textnotice_test.go"], } blueprint_go_binary { name: "xmlnotice", srcs: ["cmd/xmlnotice/xmlnotice.go"], deps: [ "compliance-module", "blueprint-deptools", ], testSrcs: ["cmd/xmlnotice/xmlnotice_test.go"], } bootstrap_go_package { name: "compliance-module", srcs: [ Loading
tools/compliance/cmd/xmlnotice/xmlnotice.go 0 → 100644 +197 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "compress/gzip" "encoding/xml" "flag" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "android/soong/tools/compliance" "github.com/google/blueprint/deptools" ) var ( outputFile = flag.String("o", "-", "Where to write the NOTICE xml or xml.gz file. (default stdout)") depsFile = flag.String("d", "", "Where to write the deps file") stripPrefix = flag.String("strip_prefix", "", "Prefix to remove from paths. i.e. path to root") failNoneRequested = fmt.Errorf("\nNo license metadata files requested") failNoLicenses = fmt.Errorf("No licenses found") ) type context struct { stdout io.Writer stderr io.Writer rootFS fs.FS stripPrefix string deps *[]string } func init() { flag.Usage = func() { fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} Outputs an xml NOTICE.xml or gzipped NOTICE.xml.gz file if the -o filename ends with ".gz". Options: `, filepath.Base(os.Args[0])) flag.PrintDefaults() } } func main() { flag.Parse() // Must specify at least one root target. if flag.NArg() == 0 { flag.Usage() os.Exit(2) } if len(*outputFile) == 0 { flag.Usage() fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") os.Exit(2) } else { dir, err := filepath.Abs(filepath.Dir(*outputFile)) if err != nil { fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err) os.Exit(1) } fi, err := os.Stat(dir) if err != nil { fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err) os.Exit(1) } if !fi.IsDir() { fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) os.Exit(1) } } var ofile io.Writer var closer io.Closer ofile = os.Stdout var obuf *bytes.Buffer if *outputFile != "-" { obuf = &bytes.Buffer{} ofile = obuf } if strings.HasSuffix(*outputFile, ".gz") { ofile, _ = gzip.NewWriterLevel(obuf, gzip.BestCompression) closer = ofile.(io.Closer) } var deps []string ctx := &context{ofile, os.Stderr, os.DirFS("."), *stripPrefix, &deps} err := xmlNotice(ctx, flag.Args()...) if err != nil { if err == failNoneRequested { flag.Usage() } fmt.Fprintf(os.Stderr, "%s\n", err.Error()) os.Exit(1) } if closer != nil { closer.Close() } if *outputFile != "-" { err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) if err != nil { fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err) os.Exit(1) } } if *depsFile != "" { err := deptools.WriteDepFile(*depsFile, *outputFile, deps) if err != nil { fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err) os.Exit(1) } } os.Exit(0) } // xmlNotice implements the xmlnotice utility. func xmlNotice(ctx *context, files ...string) error { // Must be at least one root file. if len(files) < 1 { return failNoneRequested } // Read the license graph from the license metadata files (*.meta_lic). licenseGraph, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) if err != nil { return fmt.Errorf("Unable to read license metadata file(s) %q: %v\n", files, err) } if licenseGraph == nil { return failNoLicenses } // rs contains all notice resolutions. rs := compliance.ResolveNotices(licenseGraph) ni, err := compliance.IndexLicenseTexts(ctx.rootFS, licenseGraph, rs) if err != nil { return fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) } fmt.Fprintln(ctx.stdout, "<?xml version=\"1.0\" encoding=\"utf-8\"?>") fmt.Fprintln(ctx.stdout, "<licenses>") for installPath := range ni.InstallPaths() { var p string if 0 < len(ctx.stripPrefix) && strings.HasPrefix(installPath, ctx.stripPrefix) { p = installPath[len(ctx.stripPrefix):] if 0 == len(p) { p = "root" } } else { p = installPath } for _, h := range ni.InstallHashes(installPath) { for _, lib := range ni.InstallHashLibs(installPath, h) { fmt.Fprintf(ctx.stdout, "<file-name contentId=\"%s\" lib=\"", h.String()) xml.EscapeText(ctx.stdout, []byte(lib)) fmt.Fprintf(ctx.stdout, "\">") xml.EscapeText(ctx.stdout, []byte(p)) fmt.Fprintln(ctx.stdout, "</file-name>") } } } for h := range ni.Hashes() { fmt.Fprintf(ctx.stdout, "<file-content contentId=\"%s\"><![CDATA[", h) xml.EscapeText(ctx.stdout, ni.HashText(h)) fmt.Fprintf(ctx.stdout, "]]></file-content>\n\n") } fmt.Fprintln(ctx.stdout, "</licenses>") *ctx.deps = ni.InputNoticeFiles() return nil }
tools/compliance/cmd/xmlnotice/xmlnotice_test.go 0 → 100644 +634 −0 Original line number Diff line number Diff line // Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bufio" "bytes" "encoding/xml" "fmt" "os" "reflect" "regexp" "strings" "testing" ) var ( installTarget = regexp.MustCompile(`^<file-name contentId="[^"]{32}" lib="([^"]*)">([^<]+)</file-name>`) licenseText = regexp.MustCompile(`^<file-content contentId="[^"]{32}"><![[]CDATA[[]([^]]*)[]][]]></file-content>`) ) func TestMain(m *testing.M) { // Change into the parent directory before running the tests // so they can find the testdata directory. if err := os.Chdir(".."); err != nil { fmt.Printf("failed to change to testdata directory: %s\n", err) os.Exit(1) } os.Exit(m.Run()) } func Test(t *testing.T) { tests := []struct { condition string name string roots []string stripPrefix string expectedOut []matcher expectedDeps []string }{ { condition: "firstparty", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Android"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Android"}, target{"container.zip/libb.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "firstparty", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "Android"}, firstParty{}, }, expectedDeps: []string{"testdata/firstparty/FIRST_PARTY_LICENSE"}, }, { condition: "notice", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, notice{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/notice/NOTICE_LICENSE", }, }, { condition: "notice", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "reciprocal", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", }, }, { condition: "reciprocal", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "restricted", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, restricted{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, restricted{}, reciprocal{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/reciprocal/RECIPROCAL_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "restricted", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, { condition: "proprietary", name: "apex", roots: []string{"highest.apex.meta_lic"}, expectedOut: []matcher{ target{"highest.apex", "Android"}, target{"highest.apex/bin/bin1", "Android"}, target{"highest.apex/bin/bin1", "Device"}, target{"highest.apex/bin/bin1", "External"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/bin/bin2", "Android"}, target{"highest.apex/lib/liba.so", "Device"}, target{"highest.apex/lib/libb.so", "Android"}, restricted{}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "proprietary", name: "container", roots: []string{"container.zip.meta_lic"}, expectedOut: []matcher{ target{"container.zip", "Android"}, target{"container.zip/bin1", "Android"}, target{"container.zip/bin1", "Device"}, target{"container.zip/bin1", "External"}, target{"container.zip/bin2", "Android"}, target{"container.zip/bin2", "Android"}, target{"container.zip/liba.so", "Device"}, target{"container.zip/libb.so", "Android"}, restricted{}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", "testdata/restricted/RESTRICTED_LICENSE", }, }, { condition: "proprietary", name: "application", roots: []string{"application.meta_lic"}, expectedOut: []matcher{ target{"application", "Android"}, target{"application", "Device"}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", }, }, { condition: "proprietary", name: "binary", roots: []string{"bin/bin1.meta_lic"}, expectedOut: []matcher{ target{"bin/bin1", "Android"}, target{"bin/bin1", "Device"}, target{"bin/bin1", "External"}, firstParty{}, proprietary{}, }, expectedDeps: []string{ "testdata/firstparty/FIRST_PARTY_LICENSE", "testdata/proprietary/PROPRIETARY_LICENSE", }, }, { condition: "proprietary", name: "library", roots: []string{"lib/libd.so.meta_lic"}, expectedOut: []matcher{ target{"lib/libd.so", "External"}, notice{}, }, expectedDeps: []string{"testdata/notice/NOTICE_LICENSE"}, }, } for _, tt := range tests { t.Run(tt.condition+" "+tt.name, func(t *testing.T) { stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} rootFiles := make([]string, 0, len(tt.roots)) for _, r := range tt.roots { rootFiles = append(rootFiles, "testdata/"+tt.condition+"/"+r) } var deps []string ctx := context{stdout, stderr, os.DirFS("."), tt.stripPrefix, &deps} err := xmlNotice(&ctx, rootFiles...) if err != nil { t.Fatalf("xmlnotice: error = %v, stderr = %v", err, stderr) return } if stderr.Len() > 0 { t.Errorf("xmlnotice: gotStderr = %v, want none", stderr) } t.Logf("got stdout: %s", stdout.String()) t.Logf("want stdout: %s", matcherList(tt.expectedOut).String()) out := bufio.NewScanner(stdout) lineno := 0 inBody := false outOfBody := true for out.Scan() { line := out.Text() if strings.TrimLeft(line, " ") == "" { continue } if lineno == 0 && !inBody && `<?xml version="1.0" encoding="utf-8"?>` == line { continue } if !inBody { if "<licenses>" == line { inBody = true outOfBody = false } continue } else if "</licenses>" == line { outOfBody = true continue } if len(tt.expectedOut) <= lineno { t.Errorf("xmlnotice: unexpected output at line %d: got %q, want nothing (wanted %d lines)", lineno+1, line, len(tt.expectedOut)) } else if !tt.expectedOut[lineno].isMatch(line) { t.Errorf("xmlnotice: unexpected output at line %d: got %q, want %q", lineno+1, line, tt.expectedOut[lineno].String()) } lineno++ } if !inBody { t.Errorf("xmlnotice: missing <licenses> tag: got no <licenses> tag, want <licenses> tag on 2nd line") } if !outOfBody { t.Errorf("xmlnotice: missing </licenses> tag: got no </licenses> tag, want </licenses> tag on last line") } for ; lineno < len(tt.expectedOut); lineno++ { t.Errorf("xmlnotice: missing output line %d: ended early, want %q", lineno+1, tt.expectedOut[lineno].String()) } t.Logf("got deps: %q", deps) t.Logf("want deps: %q", tt.expectedDeps) if g, w := deps, tt.expectedDeps; !reflect.DeepEqual(g, w) { t.Errorf("unexpected deps, wanted:\n%s\ngot:\n%s\n", strings.Join(w, "\n"), strings.Join(g, "\n")) } }) } } func escape(s string) string { b := &bytes.Buffer{} xml.EscapeText(b, []byte(s)) return b.String() } type matcher interface { isMatch(line string) bool String() string } type target struct { name string lib string } func (m target) isMatch(line string) bool { groups := installTarget.FindStringSubmatch(line) if len(groups) != 3 { return false } return groups[1] == escape(m.lib) && strings.HasPrefix(groups[2], "out/") && strings.HasSuffix(groups[2], "/"+escape(m.name)) } func (m target) String() string { return `<file-name contentId="hash" lib="` + escape(m.lib) + `">` + escape(m.name) + `</file-name>` } func matchesText(line, text string) bool { groups := licenseText.FindStringSubmatch(line) if len(groups) != 2 { return false } return groups[1] == escape(text + "\n") } func expectedText(text string) string { return `<file-content contentId="hash"><![CDATA[` + escape(text + "\n") + `]]></file-content>` } type firstParty struct{} func (m firstParty) isMatch(line string) bool { return matchesText(line, "&&&First Party License&&&") } func (m firstParty) String() string { return expectedText("&&&First Party License&&&") } type notice struct{} func (m notice) isMatch(line string) bool { return matchesText(line, "%%%Notice License%%%") } func (m notice) String() string { return expectedText("%%%Notice License%%%") } type reciprocal struct{} func (m reciprocal) isMatch(line string) bool { return matchesText(line, "$$$Reciprocal License$$$") } func (m reciprocal) String() string { return expectedText("$$$Reciprocal License$$$") } type restricted struct{} func (m restricted) isMatch(line string) bool { return matchesText(line, "###Restricted License###") } func (m restricted) String() string { return expectedText("###Restricted License###") } type proprietary struct{} func (m proprietary) isMatch(line string) bool { return matchesText(line, "@@@Proprietary License@@@") } func (m proprietary) String() string { return expectedText("@@@Proprietary License@@@") } type matcherList []matcher func (l matcherList) String() string { var sb strings.Builder fmt.Fprintln(&sb, `<?xml version="1.0" encoding="utf-8"?>`) fmt.Fprintln(&sb, `<licenses>`) for _, m := range l { s := m.String() fmt.Fprintln(&sb, s) if _, ok := m.(target); !ok { fmt.Fprintln(&sb) } } fmt.Fprintln(&sb, `/<licenses>`) return sb.String() }
