Merge META-INF/services/* files in merge_zips -jar (7592d5a0) · Commits · e / os / android_build_soong

cmd/merge_zips/merge_zips.go

+21 −1

Original line number	Diff line number	Diff line
		@@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 {
		}

		func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error {
		w, err := zw.CreateHeader(be.fh)
		w, err := zw.CreateHeaderAndroid(be.fh)
		if err != nil {
		return err
		}
		@@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
		}
		}

		var jarServices jar.Services

		// Finally, add entries from all the input zips.
		for _, inputZip := range inputZips {
		_, copyFully := zipsToNotStrip[inputZip.Name()]
		@@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
		}

		for i, entry := range inputZip.Entries() {
		if emulateJar && jarServices.IsServiceFile(entry) {
		// If this is a jar, collect service files to combine instead of adding them to the zip.
		err := jarServices.AddServiceFile(entry)
		if err != nil {
		return err
		}
		continue
		}
		if copyFully \|\| !out.isEntryExcluded(entry.Name) {
		if err := out.copyEntry(inputZip, i); err != nil {
		return err
		@@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string
		}

		if emulateJar {
		// Combine all the service files into a single list of combined service files and add them to the zip.
		for _, serviceFile := range jarServices.ServiceFiles() {
		_, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{
		fh: serviceFile.FileHeader,
		content: serviceFile.Contents,
		})
		if err != nil {
		return err
		}
		}
		return out.writeEntries(out.jarSorted())
		} else if sortEntries {
		return out.writeEntries(out.alphanumericSorted())

cmd/merge_zips/merge_zips_test.go

+47 −21

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@ package main
		import (
		"bytes"
		"fmt"
		"hash/crc32"
		"os"
		"strconv"
		"strings"
		@@ -30,25 +31,31 @@ type testZipEntry struct {
		name string
		mode os.FileMode
		data []byte
		method uint16
		}

		var (
		A = testZipEntry{"A", 0755, []byte("foo")}
		a = testZipEntry{"a", 0755, []byte("foo")}
		a2 = testZipEntry{"a", 0755, []byte("FOO2")}
		a3 = testZipEntry{"a", 0755, []byte("Foo3")}
		bDir = testZipEntry{"b/", os.ModeDir \| 0755, nil}
		bbDir = testZipEntry{"b/b/", os.ModeDir \| 0755, nil}
		bbb = testZipEntry{"b/b/b", 0755, nil}
		ba = testZipEntry{"b/a", 0755, []byte("foob")}
		bc = testZipEntry{"b/c", 0755, []byte("bar")}
		bd = testZipEntry{"b/d", 0700, []byte("baz")}
		be = testZipEntry{"b/e", 0700, []byte("")}

		metainfDir = testZipEntry{jar.MetaDir, os.ModeDir \| 0755, nil}
		manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")}
		manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")}
		moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")}
		A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate}
		a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate}
		a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate}
		a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate}
		bDir = testZipEntry{"b/", os.ModeDir \| 0755, nil, zip.Deflate}
		bbDir = testZipEntry{"b/b/", os.ModeDir \| 0755, nil, zip.Deflate}
		bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate}
		ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate}
		bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate}
		bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate}
		be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate}

		service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store}
		service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate}
		service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store}
		service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate}

		metainfDir = testZipEntry{jar.MetaDir, os.ModeDir \| 0755, nil, zip.Deflate}
		manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate}
		manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate}
		moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate}
		)

		type testInputZip struct {
		@@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) {
		"in1": true,
		},
		},
		{
		name: "services",
		in: [][]testZipEntry{
		{service1a, service2},
		{service1b},
		},
		jar: true,
		out: []testZipEntry{service1combined, service2},
		},
		}

		for _, test := range testCases {
		@@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) {

		closeErr := writer.Close()
		if closeErr != nil {
		t.Fatal(err)
		t.Fatal(closeErr)
		}

		if test.err != "" {
		@@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) {
		t.Fatal("incorrect err, want:", test.err, "got:", err)
		}
		return
		} else if err != nil {
		t.Fatal("unexpected err: ", err)
		}

		if !bytes.Equal(want, out.Bytes()) {
		t.Error("incorrect zip output")
		t.Errorf("want:\n%s", dumpZip(want))
		t.Errorf("got:\n%s", dumpZip(out.Bytes()))
		os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
		os.WriteFile("/tmp/want.zip", want, 0755)
		}
		})
		}
		@@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte {
		Name: e.name,
		}
		fh.SetMode(e.mode)
		fh.Method = e.method
		fh.UncompressedSize64 = uint64(len(e.data))
		fh.CRC32 = crc32.ChecksumIEEE(e.data)
		if fh.Method == zip.Store {
		fh.CompressedSize64 = fh.UncompressedSize64
		}

		w, err := zw.CreateHeader(&fh)
		w, err := zw.CreateHeaderAndroid(&fh)
		if err != nil {
		panic(err)
		}

jar/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ bootstrap_go_package {
		pkgPath: "android/soong/jar",
		srcs: [
		"jar.go",
		"services.go",
		],
		testSrcs: [
		"jar_test.go",

jar/services.go

0 → 100644

+128 −0

Original line number	Diff line number	Diff line
		// Copyright 2023 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package jar

		import (
		"android/soong/third_party/zip"
		"bufio"
		"hash/crc32"
		"sort"
		"strings"
		)

		const servicesPrefix = "META-INF/services/"

		// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing
		// the unique lines from all the input zip entries with the same name.
		type Services struct {
		services map[string]*ServiceFile
		}

		// ServiceFile contains the combined contents of all input zip entries with a single name.
		type ServiceFile struct {
		Name string
		FileHeader *zip.FileHeader
		Contents []byte
		Lines []string
		}

		// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory.
		func (Services) IsServiceFile(entry *zip.File) bool {
		return strings.HasPrefix(entry.Name, servicesPrefix)
		}

		// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need
		// to be combined.
		func (j Services) AddServiceFile(entry zip.File) error {
		if j.services == nil {
		j.services = map[string]*ServiceFile{}
		}

		service := entry.Name
		serviceFile := j.services[service]
		fh := entry.FileHeader
		if serviceFile == nil {
		serviceFile = &ServiceFile{
		Name: service,
		FileHeader: &fh,
		}
		j.services[service] = serviceFile
		}

		f, err := entry.Open()
		if err != nil {
		return err
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
		line := scanner.Text()
		if line != "" {
		serviceFile.Lines = append(serviceFile.Lines, line)
		}
		}

		if err := scanner.Err(); err != nil {
		return err
		}

		return nil
		}

		// ServiceFiles returns the list of combined service files, each containing all the unique lines from the
		// corresponding service files in the input zip entries.
		func (j *Services) ServiceFiles() []ServiceFile {
		services := make([]ServiceFile, 0, len(j.services))

		for _, serviceFile := range j.services {
		serviceFile.Lines = dedupServicesLines(serviceFile.Lines)
		serviceFile.Lines = append(serviceFile.Lines, "")
		serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n"))

		serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents))
		serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents)
		if serviceFile.FileHeader.Method == zip.Store {
		serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64
		}

		services = append(services, *serviceFile)
		}

		sort.Slice(services, func(i, j int) bool {
		return services[i].Name < services[j].Name
		})

		return services
		}

		func dedupServicesLines(in []string) []string {
		writeIndex := 0
		outer:
		for readIndex := 0; readIndex < len(in); readIndex++ {
		for compareIndex := 0; compareIndex < writeIndex; compareIndex++ {
		if interface{}(in[readIndex]) == interface{}(in[compareIndex]) {
		// The value at readIndex already exists somewhere in the output region
		// of the slice before writeIndex, skip it.
		continue outer
		}
		}
		if readIndex != writeIndex {
		in[writeIndex] = in[readIndex]
		}
		writeIndex++
		}
		return in[0:writeIndex]
		}

third_party/zip/android.go

+1 −1

Original line number	Diff line number	Diff line
		@@ -170,7 +170,7 @@ func (w Writer) CreateCompressedHeader(fh FileHeader) (io.WriteCloser, error)
		func (w Writer) CreateHeaderAndroid(fh FileHeader) (io.Writer, error) {
		writeDataDescriptor := fh.Method != Store
		if writeDataDescriptor {
		fh.Flags &= DataDescriptorFlag
		fh.Flags \|= DataDescriptorFlag
		} else {
		fh.Flags &= ^uint16(DataDescriptorFlag)
		}