Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4ccea879 authored by Jeff Sharkey's avatar Jeff Sharkey
Browse files

Custom binary XML wire protocol. 

We've identified that XML writing and reading uses roughly 1.5% of
all system_server CPU, and can generate many temporary objects.

Building on the recent TypedXmlSerializer/PullParser interfaces, this
change introduces new BinaryXmlSerializer/PullParser implementations
that store data using a custom binary wire protocol.  Benchmarking of
a typical packages.xml has shown this new binary approach can write
4.3x faster and read 8.5x faster, while using 2.4x less disk space:

    timeWrite_Fast_mean: 27946635
    timeWrite_Binary_mean: 6519341

    timeRead_Fast_mean: 59562531
    timeRead_Binary_mean: 7020185

A major factor in choosing to invest in this new wire protocol is
that it enables the long-tail of over 100 unique XML schemas used
across the OS internals to be transparently upgraded to gain these
benefits with only minimal changes, reducing the risks associated
with rewriting those schemas.

Finally, since the wire protocol is essentially a serialized event
stream, it's trivial to transparently convert this new protocol
into human-readable XML and vice-versa.  The tests in this change
demonstrate this translation working correctly, and future changes
will introduce new shell tools to aid development work.

Bug: 171832118
Test: atest FrameworksCoreTests:android.util.XmlTest
Test: atest FrameworksCoreTests:android.util.BinaryXmlTest
Test: atest CorePerfTests:android.util.XmlPerfTest
Change-Id: Ib9390701f09562dca952b3786622675b9c68a462
parent b5856365

apct-tests/perftests/core/src/android/util/XmlPerfTest.java

0 → 100644
+292 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package android.util;



import static org.junit.Assert.assertEquals;



import android.os.Bundle;

import android.os.Debug;

import android.perftests.utils.BenchmarkState;

import android.perftests.utils.PerfStatusReporter;



import androidx.test.InstrumentationRegistry;

import androidx.test.filters.LargeTest;

import androidx.test.runner.AndroidJUnit4;



import com.android.internal.util.HexDump;



import org.junit.Rule;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.xmlpull.v1.XmlPullParser;



import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.nio.charset.StandardCharsets;

import java.util.function.Supplier;



@RunWith(AndroidJUnit4.class)

@LargeTest

public class XmlPerfTest {

    /**

     * Since allocation measurement adds overhead, it's disabled by default for

     * performance runs. It can be manually enabled to compare GC behavior.

     */

    private static final boolean MEASURE_ALLOC = false;



    @Rule

    public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter();



    @Test

    public void timeWrite_Fast() throws Exception {

        doWrite(() -> Xml.newFastSerializer());

    }



    @Test

    public void timeWrite_Binary() throws Exception {

        doWrite(() -> Xml.newBinarySerializer());

    }



    private void doWrite(Supplier<TypedXmlSerializer> outFactory) throws Exception {

        if (MEASURE_ALLOC) {

            Debug.startAllocCounting();

        }



        int iterations = 0;

        final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();

        while (state.keepRunning()) {

            iterations++;

            try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {

                final TypedXmlSerializer out = outFactory.get();

                out.setOutput(os, StandardCharsets.UTF_8.name());

                write(out);

            }

        }



        if (MEASURE_ALLOC) {

            Debug.stopAllocCounting();

            final Bundle results = new Bundle();

            results.putLong("threadAllocCount_mean", Debug.getThreadAllocCount() / iterations);

            results.putLong("threadAllocSize_mean", Debug.getThreadAllocSize() / iterations);

            InstrumentationRegistry.getInstrumentation().sendStatus(0, results);

        }

    }



    @Test

    public void timeRead_Fast() throws Exception {

        doRead(() -> Xml.newFastSerializer(), () -> Xml.newFastPullParser());

    }



    @Test

    public void timeRead_Binary() throws Exception {

        doRead(() -> Xml.newBinarySerializer(), () -> Xml.newBinaryPullParser());

    }



    private void doRead(Supplier<TypedXmlSerializer> outFactory,

            Supplier<TypedXmlPullParser> inFactory) throws Exception {

        final byte[] raw;

        try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {

            TypedXmlSerializer out = outFactory.get();

            out.setOutput(os, StandardCharsets.UTF_8.name());

            write(out);

            raw = os.toByteArray();

        }



        if (MEASURE_ALLOC) {

            Debug.startAllocCounting();

        }



        int iterations = 0;

        final BenchmarkState state = mPerfStatusReporter.getBenchmarkState();

        while (state.keepRunning()) {

            iterations++;

            try (ByteArrayInputStream is = new ByteArrayInputStream(raw)) {

                TypedXmlPullParser xml = inFactory.get();

                xml.setInput(is, StandardCharsets.UTF_8.name());

                read(xml);

            }

        }



        if (MEASURE_ALLOC) {

            Debug.stopAllocCounting();

            final Bundle results = new Bundle();

            results.putLong("sizeBytes", raw.length);

            results.putLong("threadAllocCount_mean", Debug.getThreadAllocCount() / iterations);

            results.putLong("threadAllocSize_mean", Debug.getThreadAllocSize() / iterations);

            InstrumentationRegistry.getInstrumentation().sendStatus(0, results);

        } else {

            final Bundle results = new Bundle();

            results.putLong("sizeBytes", raw.length);

            InstrumentationRegistry.getInstrumentation().sendStatus(0, results);

        }

    }



    /**

     * Not even joking, this is a typical public key blob stored in

     * {@code packages.xml}.

     */

    private static final byte[] KEY_BLOB = HexDump.hexStringToByteArray(""

            + "308204a830820390a003020102020900a1573d0f45bea193300d06092a864886f70d010105050030819"

            + "4310b3009060355040613025553311330110603550408130a43616c69666f726e696131163014060355"

            + "0407130d4d6f756e7461696e20566965773110300e060355040a1307416e64726f69643110300e06035"

            + "5040b1307416e64726f69643110300e06035504031307416e64726f69643122302006092a864886f70d"

            + "0109011613616e64726f696440616e64726f69642e636f6d301e170d3131303931393138343232355a1"

            + "70d3339303230343138343232355a308194310b3009060355040613025553311330110603550408130a"

            + "43616c69666f726e6961311630140603550407130d4d6f756e7461696e20566965773110300e0603550"

            + "40a1307416e64726f69643110300e060355040b1307416e64726f69643110300e06035504031307416e"

            + "64726f69643122302006092a864886f70d0109011613616e64726f696440616e64726f69642e636f6d3"

            + "0820120300d06092a864886f70d01010105000382010d00308201080282010100de1b51336afc909d8b"

            + "cca5920fcdc8940578ec5c253898930e985481cfdea75ba6fc54b1f7bb492a03d98db471ab4200103a8"

            + "314e60ee25fef6c8b83bc1b2b45b084874cffef148fa2001bb25c672b6beba50b7ac026b546da762ea2"

            + "23829a22b80ef286131f059d2c9b4ca71d54e515a8a3fd6bf5f12a2493dfc2619b337b032a7cf8bbd34"

            + "b833f2b93aeab3d325549a93272093943bb59dfc0197ae4861ff514e019b73f5cf10023ad1a032adb4b"

            + "9bbaeb4debecb4941d6a02381f1165e1ac884c1fca9525c5854dce2ad8ec839b8ce78442c16367efc07"

            + "778a337d3ca2cdf9792ac722b95d67c345f1c00976ec372f02bfcbef0262cc512a6845e71cfea0d0201"

            + "03a381fc3081f9301d0603551d0e0416041478a0fc4517fb70ff52210df33c8d32290a44b2bb3081c90"

            + "603551d230481c13081be801478a0fc4517fb70ff52210df33c8d32290a44b2bba1819aa48197308194"

            + "310b3009060355040613025553311330110603550408130a43616c69666f726e6961311630140603550"

            + "407130d4d6f756e7461696e20566965773110300e060355040a1307416e64726f69643110300e060355"

            + "040b1307416e64726f69643110300e06035504031307416e64726f69643122302006092a864886f70d0"

            + "109011613616e64726f696440616e64726f69642e636f6d820900a1573d0f45bea193300c0603551d13"

            + "040530030101ff300d06092a864886f70d01010505000382010100977302dfbf668d7c61841c9c78d25"

            + "63bcda1b199e95e6275a799939981416909722713531157f3cdcfea94eea7bb79ca3ca972bd8058a36a"

            + "d1919291df42d7190678d4ea47a4b9552c9dfb260e6d0d9129b44615cd641c1080580e8a990dd768c6a"

            + "b500c3b964e185874e4105109d94c5bd8c405deb3cf0f7960a563bfab58169a956372167a7e2674a04c"

            + "4f80015d8f7869a7a4139aecbbdca2abc294144ee01e4109f0e47a518363cf6e9bf41f7560e94bdd4a5"

            + "d085234796b05c7a1389adfd489feec2a107955129d7991daa49afb3d327dc0dc4fe959789372b093a8"

            + "9c8dbfa41554f771c18015a6cb242a17e04d19d55d3b4664eae12caf2a11cd2b836e");



    /**

     * Typical list of permissions referenced in {@code packages.xml}.

     */

    private static final String[] PERMS = new String[] {

            "android.permission.ACCESS_CACHE_FILESYSTEM",

            "android.permission.WRITE_SETTINGS",

            "android.permission.MANAGE_EXTERNAL_STORAGE",

            "android.permission.SEND_DOWNLOAD_COMPLETED_INTENTS",

            "android.permission.FOREGROUND_SERVICE",

            "android.permission.RECEIVE_BOOT_COMPLETED",

            "android.permission.WRITE_MEDIA_STORAGE",

            "android.permission.INTERNET",

            "android.permission.UPDATE_DEVICE_STATS",

            "android.permission.RECEIVE_DEVICE_CUSTOMIZATION_READY",

            "android.permission.MANAGE_USB",

            "android.permission.ACCESS_ALL_DOWNLOADS",

            "android.permission.ACCESS_DOWNLOAD_MANAGER",

            "android.permission.MANAGE_USERS",

            "android.permission.ACCESS_NETWORK_STATE",

            "android.permission.ACCESS_MTP",

            "android.permission.INTERACT_ACROSS_USERS",

            "android.permission.CONNECTIVITY_USE_RESTRICTED_NETWORKS",

            "android.permission.CLEAR_APP_CACHE",

            "android.permission.CONNECTIVITY_INTERNAL",

            "android.permission.START_ACTIVITIES_FROM_BACKGROUND",

            "android.permission.QUERY_ALL_PACKAGES",

            "android.permission.WAKE_LOCK",

            "android.permission.UPDATE_APP_OPS_STATS",

    };



    /**

     * Write a typical {@code packages.xml} file containing 100 applications,

     * each of which defines signing key and permission information.

     */

    private static void write(TypedXmlSerializer out) throws IOException {

        out.startDocument(null, true);

        out.startTag(null, "packages");

        for (int i = 0; i < 100; i++) {

            out.startTag(null, "package");

            out.attribute(null, "name", "com.android.providers.media");

            out.attribute(null, "codePath", "/system/priv-app/MediaProviderLegacy");

            out.attribute(null, "nativeLibraryPath", "/system/priv-app/MediaProviderLegacy/lib");

            out.attributeLong(null, "publicFlags", 944258629L);

            out.attributeLong(null, "privateFlags", -1946152952L);

            out.attributeLong(null, "ft", 1603899064000L);

            out.attributeLong(null, "it", 1603899064000L);

            out.attributeLong(null, "ut", 1603899064000L);

            out.attributeInt(null, "version", 1024);

            out.attributeInt(null, "sharedUserId", 10100);

            out.attributeBoolean(null, "isOrphaned", true);



            out.startTag(null, "sigs");

            out.startTag(null, "cert");

            out.attributeInt(null, "index", 10);

            out.attributeBytesHex(null, "key", KEY_BLOB);

            out.endTag(null, "cert");

            out.endTag(null, "sigs");



            out.startTag(null, "perms");

            for (String perm : PERMS) {

                out.startTag(null, "item");

                out.attributeInterned(null, "name", perm);

                out.attributeBoolean(null, "granted", true);

                out.attributeInt(null, "flags", 0);

                out.endTag(null, "item");

            }

            out.endTag(null, "perms");



            out.endTag(null, "package");

        }

        out.endTag(null, "packages");

        out.endDocument();

    }



    /**

     * Read a typical {@code packages.xml} file containing 100 applications, and

     * verify that data passes smell test.

     */

    private static void read(TypedXmlPullParser xml) throws Exception {

        int type;

        int packages = 0;

        int certs = 0;

        int perms = 0;

        while ((type = xml.next()) != XmlPullParser.END_DOCUMENT) {

            final String tag = xml.getName();

            if (type == XmlPullParser.START_TAG) {

                if ("package".equals(tag)) {

                    xml.getAttributeValue(null, "name");

                    xml.getAttributeValue(null, "codePath");

                    xml.getAttributeValue(null, "nativeLibraryPath");

                    xml.getAttributeLong(null, "publicFlags");

                    assertEquals(-1946152952L, xml.getAttributeLong(null, "privateFlags"));

                    xml.getAttributeLong(null, "ft");

                    xml.getAttributeLong(null, "it");

                    xml.getAttributeLong(null, "ut");

                    xml.getAttributeInt(null, "version");

                    xml.getAttributeInt(null, "sharedUserId");

                    xml.getAttributeBoolean(null, "isOrphaned");

                    packages++;

                } else if ("cert".equals(tag)) {

                    xml.getAttributeInt(null, "index");

                    xml.getAttributeBytesHex(null, "key");

                    certs++;

                } else if ("item".equals(tag)) {

                    xml.getAttributeValue(null, "name");

                    xml.getAttributeBoolean(null, "granted");

                    xml.getAttributeInt(null, "flags");

                    perms++;

                }

            } else if (type == XmlPullParser.TEXT) {

                xml.getText();

            }

        }



        assertEquals(100, packages);

        assertEquals(packages * 1, certs);

        assertEquals(packages * PERMS.length, perms);

    }

}

core/java/android/util/Xml.java

+103 −0
Original line number Diff line number Diff line
@@ -17,7 +17,10 @@ 
package android.util;



import android.annotation.NonNull;

import android.annotation.Nullable;



import com.android.internal.util.BinaryXmlPullParser;

import com.android.internal.util.BinaryXmlSerializer;

import com.android.internal.util.FastXmlSerializer;

import com.android.internal.util.XmlUtils;
@@ -120,6 +123,18 @@ public class Xml { 
        return XmlUtils.makeTyped(newPullParser());

    }



    /**

     * Creates a new {@link XmlPullParser} that reads XML documents using a

     * custom binary wire protocol which benchmarking has shown to be 8.5x

     * faster than {@code Xml.newFastPullParser()} for a typical

     * {@code packages.xml}.

     *

     * @hide

     */

    public static @NonNull TypedXmlPullParser newBinaryPullParser() {

        return new BinaryXmlPullParser();

    }



    /**

     * Creates a new {@link XmlPullParser} which is optimized for use inside the

     * system, typically by supporting only a basic set of features.
@@ -166,6 +181,18 @@ public class Xml { 
        return XmlUtils.makeTyped(new FastXmlSerializer());

    }



    /**

     * Creates a new {@link XmlSerializer} that writes XML documents using a

     * custom binary wire protocol which benchmarking has shown to be 4.4x

     * faster and use 2.8x less disk space than {@code Xml.newFastSerializer()}

     * for a typical {@code packages.xml}.

     *

     * @hide

     */

    public static @NonNull TypedXmlSerializer newBinarySerializer() {

        return new BinaryXmlSerializer();

    }



    /**

     * Creates a new {@link XmlSerializer} which is optimized for use inside the

     * system, typically by supporting only a basic set of features.
@@ -188,6 +215,82 @@ public class Xml { 
        return xml;

    }



    /**

     * Copy the first XML document into the second document.

     * <p>

     * Implemented by reading all events from the given {@link XmlPullParser}

     * and writing them directly to the given {@link XmlSerializer}. This can be

     * useful for transparently converting between underlying wire protocols.

     *

     * @hide

     */

    public static void copy(@NonNull XmlPullParser in, @NonNull XmlSerializer out)

            throws XmlPullParserException, IOException {

        // Some parsers may have already consumed the event that starts the

        // document, so we manually emit that event here for consistency

        if (in.getEventType() == XmlPullParser.START_DOCUMENT) {

            out.startDocument(in.getInputEncoding(), true);

        }



        while (true) {

            final int token = in.nextToken();

            switch (token) {

                case XmlPullParser.START_DOCUMENT:

                    out.startDocument(in.getInputEncoding(), true);

                    break;

                case XmlPullParser.END_DOCUMENT:

                    out.endDocument();

                    return;

                case XmlPullParser.START_TAG:

                    out.startTag(normalizeNamespace(in.getNamespace()), in.getName());

                    for (int i = 0; i < in.getAttributeCount(); i++) {

                        out.attribute(normalizeNamespace(in.getAttributeNamespace(i)),

                                in.getAttributeName(i), in.getAttributeValue(i));

                    }

                    break;

                case XmlPullParser.END_TAG:

                    out.endTag(normalizeNamespace(in.getNamespace()), in.getName());

                    break;

                case XmlPullParser.TEXT:

                    out.text(in.getText());

                    break;

                case XmlPullParser.CDSECT:

                    out.cdsect(in.getText());

                    break;

                case XmlPullParser.ENTITY_REF:

                    out.entityRef(in.getName());

                    break;

                case XmlPullParser.IGNORABLE_WHITESPACE:

                    out.ignorableWhitespace(in.getText());

                    break;

                case XmlPullParser.PROCESSING_INSTRUCTION:

                    out.processingInstruction(in.getText());

                    break;

                case XmlPullParser.COMMENT:

                    out.comment(in.getText());

                    break;

                case XmlPullParser.DOCDECL:

                    out.docdecl(in.getText());

                    break;

                default:

                    throw new IllegalStateException("Unknown token " + token);

            }

        }

    }



    /**

     * Some parsers may return an empty string {@code ""} when a namespace in

     * unsupported, which can confuse serializers. This method normalizes empty

     * strings to be {@code null}.

     */

    private static @Nullable String normalizeNamespace(@Nullable String namespace) {

        if (namespace == null || namespace.isEmpty()) {

            return null;

        } else {

            return namespace;

        }

    }



    /**

     * Supported character encodings.

     */

core/java/com/android/internal/util/BinaryXmlPullParser.java

0 → 100644
+899 −0

File added.

Preview size limit exceeded, changes collapsed.

core/java/com/android/internal/util/BinaryXmlSerializer.java

0 → 100644
+396 −0

File added.

Preview size limit exceeded, changes collapsed.

core/tests/coretests/src/android/util/BinaryXmlTest.java

0 → 100644
+99 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2020 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package android.util;



import static android.util.XmlTest.assertNext;

import static android.util.XmlTest.buildPersistableBundle;

import static android.util.XmlTest.doPersistableBundleRead;

import static android.util.XmlTest.doPersistableBundleWrite;



import static org.junit.Assert.assertEquals;

import static org.xmlpull.v1.XmlPullParser.START_TAG;



import android.os.PersistableBundle;



import androidx.test.runner.AndroidJUnit4;



import org.junit.Test;

import org.junit.runner.RunWith;



import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.nio.charset.StandardCharsets;



@RunWith(AndroidJUnit4.class)

public class BinaryXmlTest {

    /**

     * Verify that we can write and read large numbers of interned

     * {@link String} values.

     */

    @Test

    public void testLargeInterned_Binary() throws Exception {

        // We're okay with the tag itself being interned

        final int count = (1 << 16) - 2;



        final TypedXmlSerializer out = Xml.newBinarySerializer();

        final ByteArrayOutputStream os = new ByteArrayOutputStream();

        out.setOutput(os, StandardCharsets.UTF_8.name());

        out.startTag(null, "tag");

        for (int i = 0; i < count; i++) {

            out.attribute(null, "name" + i, "value");

        }

        out.endTag(null, "tag");

        out.flush();



        final TypedXmlPullParser in = Xml.newBinaryPullParser();

        final ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray());

        in.setInput(is, StandardCharsets.UTF_8.name());

        assertNext(in, START_TAG, "tag", 1);

        assertEquals(count, in.getAttributeCount());

    }



    @Test

    public void testTranscode_FastToBinary() throws Exception {

        doTranscode(Xml.newFastSerializer(), Xml.newFastPullParser(),

                Xml.newBinarySerializer(), Xml.newBinaryPullParser());

    }



    @Test

    public void testTranscode_BinaryToFast() throws Exception {

        doTranscode(Xml.newBinarySerializer(), Xml.newBinaryPullParser(),

                Xml.newFastSerializer(), Xml.newFastPullParser());

    }



    /**

     * Verify that a complex {@link PersistableBundle} can be transcoded using

     * the two given formats with the original structure intact.

     */

    private static void doTranscode(TypedXmlSerializer firstOut, TypedXmlPullParser firstIn,

            TypedXmlSerializer secondOut, TypedXmlPullParser secondIn) throws Exception {

        final PersistableBundle expected = buildPersistableBundle();

        final byte[] firstRaw = doPersistableBundleWrite(firstOut, expected);



        // Perform actual transcoding between the two formats

        final ByteArrayInputStream is = new ByteArrayInputStream(firstRaw);

        firstIn.setInput(is, StandardCharsets.UTF_8.name());

        final ByteArrayOutputStream os = new ByteArrayOutputStream();

        secondOut.setOutput(os, StandardCharsets.UTF_8.name());

        Xml.copy(firstIn, secondOut);



        // Yes, this string-based check is fragile, but kindofEquals() is broken

        // when working with nested objects and arrays

        final PersistableBundle actual = doPersistableBundleRead(secondIn, os.toByteArray());

        assertEquals(expected.toString(), actual.toString());

    }

}