Merge "Support moving sources in srcjars in soong_zip" (7dc6951c) · Commits · e / os / android_build_soong

jar/Android.bp

+3 −1

Original line number	Diff line number	Diff line
		@@ -18,8 +18,10 @@ bootstrap_go_package {
		srcs: [
		"jar.go",
		],
		testSrcs: [
		"jar_test.go",
		],
		deps: [
		"android-archive-zip",
		],
		}

jar/jar.go

+111 −0

Original line number	Diff line number	Diff line
		@@ -17,9 +17,12 @@ package jar
		import (
		"bytes"
		"fmt"
		"io"
		"os"
		"strings"
		"text/scanner"
		"time"
		"unicode"

		"android/soong/third_party/zip"
		)
		@@ -112,3 +115,111 @@ func manifestContents(contents []byte) ([]byte, error) {

		return finalBytes, nil
		}

		var javaIgnorableIdentifier = &unicode.RangeTable{
		R16: []unicode.Range16{
		{0x00, 0x08, 1},
		{0x0e, 0x1b, 1},
		{0x7f, 0x9f, 1},
		},
		LatinOffset: 3,
		}

		func javaIdentRune(ch rune, i int) bool {
		if unicode.IsLetter(ch) {
		return true
		}
		if unicode.IsDigit(ch) && i > 0 {
		return true
		}

		if unicode.In(ch,
		unicode.Nl, // letter number
		unicode.Sc, // currency symbol
		unicode.Pc, // connecting punctuation
		) {
		return true
		}

		if unicode.In(ch,
		unicode.Cf, // format
		unicode.Mc, // combining mark
		unicode.Mn, // non-spacing mark
		javaIgnorableIdentifier,
		) && i > 0 {
		return true
		}

		return false
		}

		// JavaPackage parses the package out of a java source file by looking for the package statement, or the first valid
		// non-package statement, in which case it returns an empty string for the package.
		func JavaPackage(r io.Reader, src string) (string, error) {
		var s scanner.Scanner
		var sErr error

		s.Init(r)
		s.Filename = src
		s.Error = func(s *scanner.Scanner, msg string) {
		sErr = fmt.Errorf("error parsing %q: %s", src, msg)
		}
		s.IsIdentRune = javaIdentRune

		tok := s.Scan()
		if sErr != nil {
		return "", sErr
		}
		if tok == scanner.Ident {
		switch s.TokenText() {
		case "package":
		// Nothing
		case "import":
		// File has no package statement, first keyword is an import
		return "", nil
		case "class", "enum", "interface":
		// File has no package statement, first keyword is a type declaration
		return "", nil
		case "public", "protected", "private", "abstract", "static", "final", "strictfp":
		// File has no package statement, first keyword is a modifier
		return "", nil
		case "module", "open":
		// File has no package statement, first keyword is a module declaration
		return "", nil
		default:
		return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
		}
		} else if tok == '@' {
		// File has no package statement, first token is an annotation
		return "", nil
		} else if tok == scanner.EOF {
		// File no package statement, it has no non-whitespace non-comment tokens
		return "", nil
		} else {
		return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
		}

		var pkg string
		for {
		tok = s.Scan()
		if sErr != nil {
		return "", sErr
		}
		if tok != scanner.Ident {
		return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
		}
		pkg += s.TokenText()

		tok = s.Scan()
		if sErr != nil {
		return "", sErr
		}
		if tok == ';' {
		return pkg, nil
		} else if tok == '.' {
		pkg += "."
		} else {
		return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
		}
		}
		}

jar/jar_test.go

0 → 100644

+182 −0

Original line number	Diff line number	Diff line
		// Copyright 2017 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package jar

		import (
		"bytes"
		"io"
		"testing"
		)

		func TestGetJavaPackage(t *testing.T) {
		type args struct {
		r io.Reader
		src string
		}
		tests := []struct {
		name string
		in string
		want string
		wantErr bool
		}{
		{
		name: "simple",
		in: "package foo.bar;",
		want: "foo.bar",
		},
		{
		name: "comment",
		in: "/* test */\npackage foo.bar;",
		want: "foo.bar",
		},
		{
		name: "no package",
		in: "import foo.bar;",
		want: "",
		},
		{
		name: "missing semicolon error",
		in: "package foo.bar",
		wantErr: true,
		},
		{
		name: "parser error",
		in: "/*",
		wantErr: true,
		},
		{
		name: "parser ident error",
		in: "package 0foo.bar;",
		wantErr: true,
		},
		}
		for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
		buf := bytes.NewBufferString(tt.in)
		got, err := JavaPackage(buf, "<test>")
		if (err != nil) != tt.wantErr {
		t.Errorf("JavaPackage() error = %v, wantErr %v", err, tt.wantErr)
		return
		}
		if got != tt.want {
		t.Errorf("JavaPackage() = %v, want %v", got, tt.want)
		}
		})
		}
		}

		func Test_javaIdentRune(t *testing.T) {
		// runes that should be valid anywhere in an identifier
		validAnywhere := []rune{
		// letters, $, _
		'a',
		'A',
		'$',
		'_',

		// assorted unicode
		'𐐀',
		'𐐨',
		'ǅ',
		'ῼ',
		'ʰ',
		'ﾟ',
		'ƻ',
		'㡢',
		'￦',
		'＿',
		'Ⅰ',
		'𐍊',
		}

		// runes that should be invalid as the first rune in an identifier, but valid anywhere else
		validAfterFirst := []rune{
		// digits
		'0',

		// assorted unicode
		'᥍',
		'𝟎',
		'ྂ',
		'𝆀',

		// control characters
		'\x00',
		'\b',
		'\u000e',
		'\u001b',
		'\u007f',
		'\u009f',
		'\u00ad',
		0xE007F,

		// zero width space
		'\u200b',
		}

		// runes that should never be valid in an identifier
		invalid := []rune{
		';',
		0x110000,
		}

		validFirst := validAnywhere
		invalidFirst := append(validAfterFirst, invalid...)
		validPart := append(validAnywhere, validAfterFirst...)
		invalidPart := invalid

		check := func(t *testing.T, ch rune, i int, want bool) {
		t.Helper()
		if got := javaIdentRune(ch, i); got != want {
		t.Errorf("javaIdentRune() = %v, want %v", got, want)
		}
		}

		t.Run("first", func(t *testing.T) {
		t.Run("valid", func(t *testing.T) {
		for _, ch := range validFirst {
		t.Run(string(ch), func(t *testing.T) {
		check(t, ch, 0, true)
		})
		}
		})

		t.Run("invalid", func(t *testing.T) {
		for _, ch := range invalidFirst {
		t.Run(string(ch), func(t *testing.T) {
		check(t, ch, 0, false)
		})
		}
		})
		})

		t.Run("part", func(t *testing.T) {
		t.Run("valid", func(t *testing.T) {
		for _, ch := range validPart {
		t.Run(string(ch), func(t *testing.T) {
		check(t, ch, 1, true)
		})
		}
		})

		t.Run("invalid", func(t *testing.T) {
		for _, ch := range invalidPart {
		t.Run(string(ch), func(t *testing.T) {
		check(t, ch, 1, false)
		})
		}
		})
		})
		}

zip/cmd/main.go

+2 −0

Original line number	Diff line number	Diff line
		@@ -136,6 +136,7 @@ func main() {
		writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
		ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
		symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
		srcJar := flags.Bool("srcjar", false, "move .java files to locations that match their package statement")

		parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
		cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file")
		@@ -191,6 +192,7 @@ func main() {
		FileArgs: fileArgsBuilder.FileArgs(),
		OutputFilePath: *out,
		EmulateJar: *emulateJar,
		SrcJar: *srcJar,
		AddDirectoryEntriesToZip: *directories,
		CompressionLevel: *compLevel,
		ManifestSourcePath: *manifest,

zip/zip.go

+60 −28

Original line number	Diff line number	Diff line
		@@ -210,6 +210,7 @@ type ZipArgs struct {
		FileArgs []FileArg
		OutputFilePath string
		EmulateJar bool
		SrcJar bool
		AddDirectoryEntriesToZip bool
		CompressionLevel int
		ManifestSourcePath string
		@@ -364,7 +365,7 @@ func ZipTo(args ZipArgs, w io.Writer) error {
		}
		}

		return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
		return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
		}

		func Zip(args ZipArgs) error {
		@@ -446,7 +447,9 @@ func jarSort(mappings []pathMapping) {
		sort.SliceStable(mappings, less)
		}

		func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
		func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
		parallelJobs int) error {

		z.errors = make(chan error)
		defer close(z.errors)

		@@ -489,7 +492,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
		if emulateJar && ele.dest == jar.ManifestFile {
		err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
		} else {
		err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
		err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
		}
		if err != nil {
		z.errors <- err
		@@ -588,7 +591,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
		}

		// imports (possibly with compression) <src> into the zip at sub-path <dest>
		func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
		func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
		var fileSize int64
		var executable bool

		@@ -606,12 +609,9 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
		return nil
		}
		return err
		} else if s.IsDir() {
		if z.directories {
		return z.writeDirectory(dest, src, emulateJar)
		}
		return nil
		} else {

		createParentDirs := func(dest, src string) error {
		if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
		return err
		}
		@@ -625,21 +625,45 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er

		z.createdFiles[dest] = src

		if s.Mode()&os.ModeSymlink != 0 {
		return z.writeSymlink(dest, src)
		} else if !s.Mode().IsRegular() {
		return fmt.Errorf("%s is not a file, directory, or symlink", src)
		return nil
		}

		fileSize = s.Size()
		executable = s.Mode()&0100 != 0
		if s.IsDir() {
		if z.directories {
		return z.writeDirectory(dest, src, emulateJar)
		}
		return nil
		} else if s.Mode()&os.ModeSymlink != 0 {
		err = createParentDirs(dest, src)
		if err != nil {
		return err
		}

		return z.writeSymlink(dest, src)
		} else if s.Mode().IsRegular() {
		r, err := z.fs.Open(src)
		if err != nil {
		return err
		}

		if srcJar && filepath.Ext(src) == ".java" {
		// rewrite the destination using the package path if it can be determined
		pkg, err := jar.JavaPackage(r, src)
		if err != nil {
		// ignore errors for now, leaving the file at in its original location in the zip
		} else {
		dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
		}

		_, err = r.Seek(0, io.SeekStart)
		if err != nil {
		return err
		}
		}

		fileSize = s.Size()
		executable = s.Mode()&0100 != 0

		header := &zip.FileHeader{
		Name: dest,
		Method: method,
		@@ -650,7 +674,15 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
		header.SetMode(0700)
		}

		err = createParentDirs(dest, src)
		if err != nil {
		return err
		}

		return z.writeFileContents(header, r)
		} else {
		return fmt.Errorf("%s is not a file, directory, or symlink", src)
		}
		}

		func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {