Merge changes Ie7f776a4,I3ca5dd1f,I2e911262 (89d99337) · Commits · e / os / android_build_soong

cmd/diff_target_files/Android.bp

0 → 100644

+16 −0

Original line number	Diff line number	Diff line
		blueprint_go_binary {
		name: "diff_target_files",
		srcs: [
		"compare.go",
		"diff_target_files.go",
		"glob.go",
		"target_files.go",
		"whitelist.go",
		"zip_artifact.go",
		],
		testSrcs: [
		"compare_test.go",
		"glob_test.go",
		"whitelist_test.go",
		],
		}

cmd/diff_target_files/compare.go

0 → 100644

+133 −0

Original line number	Diff line number	Diff line
		// Copyright 2019 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"bytes"
		"fmt"
		)

		// compareTargetFiles takes two ZipArtifacts and compares the files they contain by examining
		// the path, size, and CRC of each file.
		func compareTargetFiles(priZip, refZip ZipArtifact, artifact string, whitelists []whitelist, filters []string) (zipDiff, error) {
		priZipFiles, err := priZip.Files()
		if err != nil {
		return zipDiff{}, fmt.Errorf("error fetching target file lists from primary zip %v", err)
		}

		refZipFiles, err := refZip.Files()
		if err != nil {
		return zipDiff{}, fmt.Errorf("error fetching target file lists from reference zip %v", err)
		}

		priZipFiles, err = filterTargetZipFiles(priZipFiles, artifact, filters)
		if err != nil {
		return zipDiff{}, err
		}

		refZipFiles, err = filterTargetZipFiles(refZipFiles, artifact, filters)
		if err != nil {
		return zipDiff{}, err
		}

		// Compare the file lists from both builds
		diff := diffTargetFilesLists(refZipFiles, priZipFiles)

		return applyWhitelists(diff, whitelists)
		}

		// zipDiff contains the list of files that differ between two zip files.
		type zipDiff struct {
		modified [][2]*ZipArtifactFile
		onlyInA, onlyInB []*ZipArtifactFile
		}

		// String pretty-prints the list of files that differ between two zip files.
		func (d *zipDiff) String() string {
		buf := &bytes.Buffer{}

		must := func(n int, err error) {
		if err != nil {
		panic(err)
		}
		}

		var sizeChange int64

		if len(d.modified) > 0 {
		must(fmt.Fprintln(buf, "files modified:"))
		for _, f := range d.modified {
		must(fmt.Fprintf(buf, " %v (%v bytes -> %v bytes)\n", f[0].Name, f[0].UncompressedSize64, f[1].UncompressedSize64))
		sizeChange += int64(f[1].UncompressedSize64) - int64(f[0].UncompressedSize64)
		}
		}

		if len(d.onlyInA) > 0 {
		must(fmt.Fprintln(buf, "files removed:"))
		for _, f := range d.onlyInA {
		must(fmt.Fprintf(buf, " - %v (%v bytes)\n", f.Name, f.UncompressedSize64))
		sizeChange -= int64(f.UncompressedSize64)
		}
		}

		if len(d.onlyInB) > 0 {
		must(fmt.Fprintln(buf, "files added:"))
		for _, f := range d.onlyInB {
		must(fmt.Fprintf(buf, " + %v (%v bytes)\n", f.Name, f.UncompressedSize64))
		sizeChange += int64(f.UncompressedSize64)
		}
		}

		if len(d.modified) > 0 \|\| len(d.onlyInA) > 0 \|\| len(d.onlyInB) > 0 {
		must(fmt.Fprintf(buf, "total size change: %v bytes\n", sizeChange))
		}

		return buf.String()
		}

		func diffTargetFilesLists(a, b []*ZipArtifactFile) zipDiff {
		i := 0
		j := 0

		diff := zipDiff{}

		for i < len(a) && j < len(b) {
		if a[i].Name == b[j].Name {
		if a[i].UncompressedSize64 != b[j].UncompressedSize64 \|\| a[i].CRC32 != b[j].CRC32 {
		diff.modified = append(diff.modified, [2]*ZipArtifactFile{a[i], b[j]})
		}
		i++
		j++
		} else if a[i].Name < b[j].Name {
		// a[i] is not present in b
		diff.onlyInA = append(diff.onlyInA, a[i])
		i++
		} else {
		// b[j] is not present in a
		diff.onlyInB = append(diff.onlyInB, b[j])
		j++
		}
		}
		for i < len(a) {
		diff.onlyInA = append(diff.onlyInA, a[i])
		i++
		}
		for j < len(b) {
		diff.onlyInB = append(diff.onlyInB, b[j])
		j++
		}

		return diff
		}

cmd/diff_target_files/compare_test.go

0 → 100644

+131 −0

Original line number	Diff line number	Diff line
		// Copyright 2019 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"archive/zip"
		"reflect"
		"testing"
		)

		func TestDiffTargetFilesLists(t *testing.T) {
		zipArtifactFile := func(name string, crc32 uint32, size uint64) *ZipArtifactFile {
		return &ZipArtifactFile{
		File: &zip.File{
		FileHeader: zip.FileHeader{
		Name: name,
		CRC32: crc32,
		UncompressedSize64: size,
		},
		},
		}
		}
		x0 := zipArtifactFile("x", 0, 0)
		x1 := zipArtifactFile("x", 1, 0)
		x2 := zipArtifactFile("x", 0, 2)
		y0 := zipArtifactFile("y", 0, 0)
		//y1 := zipArtifactFile("y", 1, 0)
		//y2 := zipArtifactFile("y", 1, 2)
		z0 := zipArtifactFile("z", 0, 0)
		z1 := zipArtifactFile("z", 1, 0)
		//z2 := zipArtifactFile("z", 1, 2)

		testCases := []struct {
		name string
		a, b []*ZipArtifactFile
		diff zipDiff
		}{
		{
		name: "same",
		a: []*ZipArtifactFile{x0, y0, z0},
		b: []*ZipArtifactFile{x0, y0, z0},
		diff: zipDiff{nil, nil, nil},
		},
		{
		name: "first only in a",
		a: []*ZipArtifactFile{x0, y0, z0},
		b: []*ZipArtifactFile{y0, z0},
		diff: zipDiff{nil, []*ZipArtifactFile{x0}, nil},
		},
		{
		name: "middle only in a",
		a: []*ZipArtifactFile{x0, y0, z0},
		b: []*ZipArtifactFile{x0, z0},
		diff: zipDiff{nil, []*ZipArtifactFile{y0}, nil},
		},
		{
		name: "last only in a",
		a: []*ZipArtifactFile{x0, y0, z0},
		b: []*ZipArtifactFile{x0, y0},
		diff: zipDiff{nil, []*ZipArtifactFile{z0}, nil},
		},

		{
		name: "first only in b",
		a: []*ZipArtifactFile{y0, z0},
		b: []*ZipArtifactFile{x0, y0, z0},
		diff: zipDiff{nil, nil, []*ZipArtifactFile{x0}},
		},
		{
		name: "middle only in b",
		a: []*ZipArtifactFile{x0, z0},
		b: []*ZipArtifactFile{x0, y0, z0},
		diff: zipDiff{nil, nil, []*ZipArtifactFile{y0}},
		},
		{
		name: "last only in b",
		a: []*ZipArtifactFile{x0, y0},
		b: []*ZipArtifactFile{x0, y0, z0},
		diff: zipDiff{nil, nil, []*ZipArtifactFile{z0}},
		},

		{
		name: "diff",
		a: []*ZipArtifactFile{x0},
		b: []*ZipArtifactFile{x1},
		diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, nil, nil},
		},
		{
		name: "diff plus unique last",
		a: []*ZipArtifactFile{x0, y0},
		b: []*ZipArtifactFile{x1, z0},
		diff: zipDiff{[][2]ZipArtifactFile{{x0, x1}}, []ZipArtifactFile{y0}, []*ZipArtifactFile{z0}},
		},
		{
		name: "diff plus unique first",
		a: []*ZipArtifactFile{x0, z0},
		b: []*ZipArtifactFile{y0, z1},
		diff: zipDiff{[][2]ZipArtifactFile{{z0, z1}}, []ZipArtifactFile{x0}, []*ZipArtifactFile{y0}},
		},
		{
		name: "diff size",
		a: []*ZipArtifactFile{x0},
		b: []*ZipArtifactFile{x2},
		diff: zipDiff{[][2]*ZipArtifactFile{{x0, x2}}, nil, nil},
		},
		}

		for _, test := range testCases {
		t.Run(test.name, func(t *testing.T) {
		diff := diffTargetFilesLists(test.a, test.b)

		if !reflect.DeepEqual(diff, test.diff) {

		t.Errorf("diffTargetFilesLists = %v, %v, %v", diff.modified, diff.onlyInA, diff.onlyInB)
		t.Errorf(" want %v, %v, %v", test.diff.modified, test.diff.onlyInA, test.diff.onlyInB)
		}
		})
		}
		}

cmd/diff_target_files/diff_target_files.go

0 → 100644

+82 −0

Original line number	Diff line number	Diff line
		// Copyright 2019 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"flag"
		"fmt"
		"os"
		"strings"
		)

		var (
		whitelists = newMultiString("whitelist", "whitelist patterns in the form <pattern>[:<regex of line to ignore>]")
		whitelistFiles = newMultiString("whitelist_file", "files containing whitelist definitions")

		filters = newMultiString("filter", "filter patterns to apply to files in target-files.zip before comparing")
		)

		func newMultiString(name, usage string) *multiString {
		var f multiString
		flag.Var(&f, name, usage)
		return &f
		}

		type multiString []string

		func (ms multiString) String() string { return strings.Join(ms, ", ") }
		func (ms multiString) Set(s string) error { ms = append(*ms, s); return nil }

		func main() {
		flag.Parse()

		if flag.NArg() != 2 {
		fmt.Fprintf(os.Stderr, "Error, exactly two arguments are required\n")
		os.Exit(1)
		}

		whitelists, err := parseWhitelists(whitelists, whitelistFiles)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error parsing whitelists: %v\n", err)
		os.Exit(1)
		}

		priZip, err := NewLocalZipArtifact(flag.Arg(0))
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error opening zip file %v: %v\n", flag.Arg(0), err)
		os.Exit(1)
		}
		defer priZip.Close()

		refZip, err := NewLocalZipArtifact(flag.Arg(1))
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error opening zip file %v: %v\n", flag.Arg(1), err)
		os.Exit(1)
		}
		defer refZip.Close()

		diff, err := compareTargetFiles(priZip, refZip, targetFilesPattern, whitelists, *filters)
		if err != nil {
		fmt.Fprintf(os.Stderr, "Error comparing zip files: %v\n", err)
		os.Exit(1)
		}

		fmt.Print(diff.String())

		if len(diff.modified) > 0 \|\| len(diff.onlyInA) > 0 \|\| len(diff.onlyInB) > 0 {
		fmt.Fprintln(os.Stderr, "differences found")
		os.Exit(1)
		}
		}

cmd/diff_target_files/glob.go

0 → 100644

+81 −0

Original line number	Diff line number	Diff line
		// Copyright 2019 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"errors"
		"path/filepath"
		"strings"
		)

		// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
		// recursive globs (**).
		func Match(pattern, name string) (bool, error) {
		if filepath.Base(pattern) == "**" {
		return false, errors.New("pattern has '**' as last path element")
		}

		patternDir := pattern[len(pattern)-1] == '/'
		nameDir := name[len(name)-1] == '/'

		if patternDir != nameDir {
		return false, nil
		}

		if nameDir {
		name = name[:len(name)-1]
		pattern = pattern[:len(pattern)-1]
		}

		for {
		var patternFile, nameFile string
		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)

		if patternFile == "**" {
		if strings.Contains(pattern, "**") {
		return false, errors.New("pattern contains multiple '**'")
		}
		// Test if the any prefix of name matches the part of the pattern before **
		for {
		if name == "." \|\| name == "/" {
		return name == pattern, nil
		}
		if match, err := filepath.Match(pattern, name); err != nil {
		return false, err
		} else if match {
		return true, nil
		}
		name = filepath.Dir(name)
		}
		} else if strings.Contains(patternFile, "**") {
		return false, errors.New("pattern contains other characters between '**' and path separator")
		}

		name, nameFile = filepath.Dir(name), filepath.Base(name)

		if nameFile == "." && patternFile == "." {
		return true, nil
		} else if nameFile == "/" && patternFile == "/" {
		return true, nil
		} else if nameFile == "." \|\| patternFile == "." \|\| nameFile == "/" \|\| patternFile == "/" {
		return false, nil
		}

		match, err := filepath.Match(patternFile, nameFile)
		if err != nil \|\| !match {
		return match, err
		}
		}
		}