Merge "v4 digest tree streaming" into rvc-dev (437fd07a) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/pm/PackageManagerShellCommand.java

+26 −13

Original line number	Diff line number	Diff line
		@@ -105,6 +105,7 @@ import com.android.internal.util.ArrayUtils;
		import com.android.internal.util.IndentingPrintWriter;
		import com.android.server.LocalServices;
		import com.android.server.SystemConfig;
		import com.android.server.pm.PackageManagerShellCommandDataLoader.Metadata;

		import dalvik.system.DexFile;

		@@ -118,7 +119,6 @@ import java.io.InputStream;
		import java.io.OutputStream;
		import java.io.PrintWriter;
		import java.net.URISyntaxException;
		import java.nio.charset.StandardCharsets;
		import java.util.ArrayList;
		import java.util.Base64;
		import java.util.Collection;
		@@ -3025,9 +3025,9 @@ class PackageManagerShellCommand extends ShellCommand {
		// 1. Single file from stdin.
		if (args.isEmpty() \|\| STDIN_PATH.equals(args.get(0))) {
		final String name = "base." + (isApex ? "apex" : "apk");
		final String metadata = "-" + name;
		final Metadata metadata = Metadata.forStdIn(name);
		session.addFile(LOCATION_DATA_APP, name, sessionSizeBytes,
		metadata.getBytes(StandardCharsets.UTF_8), null);
		metadata.toByteArray(), null);
		return 0;
		}

		@@ -3056,9 +3056,10 @@ class PackageManagerShellCommand extends ShellCommand {

		private int processArgForStdin(String arg, PackageInstaller.Session session) {
		final String[] fileDesc = arg.split(":");
		String name, metadata;
		String name, fileId;
		long sizeBytes;
		byte[] signature = null;
		int streamingVersion = 0;

		try {
		if (fileDesc.length < 2) {
		@@ -3067,14 +3068,22 @@ class PackageManagerShellCommand extends ShellCommand {
		}
		name = fileDesc[0];
		sizeBytes = Long.parseUnsignedLong(fileDesc[1]);
		metadata = name;
		fileId = name;

		if (fileDesc.length > 2 && !TextUtils.isEmpty(fileDesc[2])) {
		metadata = fileDesc[2];
		fileId = fileDesc[2];
		}
		if (fileDesc.length > 3) {
		signature = Base64.getDecoder().decode(fileDesc[3]);
		}
		if (fileDesc.length > 4) {
		streamingVersion = Integer.parseUnsignedInt(fileDesc[4]);
		if (streamingVersion < 0 \|\| streamingVersion > 1) {
		getErrPrintWriter().println(
		"Unsupported streaming version: " + streamingVersion);
		return 1;
		}
		}
		} catch (IllegalArgumentException e) {
		getErrPrintWriter().println(
		"Unable to parse file parameters: " + arg + ", reason: " + e);
		@@ -3086,9 +3095,14 @@ class PackageManagerShellCommand extends ShellCommand {
		return 1;
		}

		final Metadata metadata;

		if (signature != null) {
		// Streaming/adb mode.
		metadata = "+" + metadata;
		// Streaming/adb mode. Versions:
		// 0: data only streaming, tree has to be fully available,
		// 1: tree and data streaming.
		metadata = (streamingVersion == 0) ? Metadata.forDataOnlyStreaming(fileId)
		: Metadata.forStreaming(fileId);
		try {
		if (V4Signature.readFrom(signature) == null) {
		getErrPrintWriter().println("V4 signature is invalid in: " + arg);
		@@ -3101,11 +3115,10 @@ class PackageManagerShellCommand extends ShellCommand {
		}
		} else {
		// Single-shot read from stdin.
		metadata = "-" + metadata;
		metadata = Metadata.forStdIn(fileId);
		}

		session.addFile(LOCATION_DATA_APP, name, sizeBytes,
		metadata.getBytes(StandardCharsets.UTF_8), signature);
		session.addFile(LOCATION_DATA_APP, name, sizeBytes, metadata.toByteArray(), signature);
		return 0;
		}

		@@ -3115,7 +3128,7 @@ class PackageManagerShellCommand extends ShellCommand {
		final File file = new File(inPath);
		final String name = file.getName();
		final long size = file.length();
		final byte[] metadata = inPath.getBytes(StandardCharsets.UTF_8);
		final Metadata metadata = Metadata.forLocalFile(inPath);

		byte[] v4signatureBytes = null;
		// Try to load the v4 signature file for the APK; it might not exist.
		@@ -3132,7 +3145,7 @@ class PackageManagerShellCommand extends ShellCommand {
		}
		}

		session.addFile(LOCATION_DATA_APP, name, size, metadata, v4signatureBytes);
		session.addFile(LOCATION_DATA_APP, name, size, metadata.toByteArray(), v4signatureBytes);
		}

		private int doWriteSplits(int sessionId, ArrayList<String> splitPaths, long sessionSizeBytes,

services/core/java/com/android/server/pm/PackageManagerShellCommandDataLoader.java

+92 −13

Original line number	Diff line number	Diff line
		@@ -24,7 +24,6 @@ import android.content.pm.PackageInstaller;
		import android.os.ParcelFileDescriptor;
		import android.os.ShellCommand;
		import android.service.dataloader.DataLoaderService;
		import android.text.TextUtils;
		import android.util.Slog;
		import android.util.SparseArray;

		@@ -114,6 +113,74 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
		}
		}

		static class Metadata {
		/**
		* Full files read from stdin.
		*/
		static final byte STDIN = 0;
		/**
		* Full files read from local file.
		*/
		static final byte LOCAL_FILE = 1;
		/**
		* Signature tree read from stdin, data streamed.
		*/
		static final byte DATA_ONLY_STREAMING = 2;
		/**
		* Everything streamed.
		*/
		static final byte STREAMING = 3;

		private final byte mMode;
		private final String mData;

		static Metadata forStdIn(String fileId) {
		return new Metadata(STDIN, fileId);
		}

		static Metadata forLocalFile(String filePath) {
		return new Metadata(LOCAL_FILE, filePath);
		}

		static Metadata forDataOnlyStreaming(String fileId) {
		return new Metadata(DATA_ONLY_STREAMING, fileId);
		}

		static Metadata forStreaming(String fileId) {
		return new Metadata(STREAMING, fileId);
		}

		private Metadata(byte mode, String data) {
		this.mMode = mode;
		this.mData = (data == null) ? "" : data;
		}

		static Metadata fromByteArray(byte[] bytes) throws IOException {
		if (bytes == null \|\| bytes.length == 0) {
		return null;
		}
		byte mode = bytes[0];
		String data = new String(bytes, 1, bytes.length - 1, StandardCharsets.UTF_8);
		return new Metadata(mode, data);
		}

		byte[] toByteArray() {
		byte[] dataBytes = this.mData.getBytes(StandardCharsets.UTF_8);
		byte[] result = new byte[1 + dataBytes.length];
		result[0] = this.mMode;
		System.arraycopy(dataBytes, 0, result, 1, dataBytes.length);
		return result;
		}

		byte getMode() {
		return this.mMode;
		}

		String getData() {
		return this.mData;
		}
		}

		private static class DataLoader implements DataLoaderService.DataLoader {
		private DataLoaderParams mParams = null;
		private FileSystemConnector mConnector = null;
		@@ -136,19 +203,31 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
		}
		try {
		for (InstallationFile file : addedFiles) {
		String filePath = new String(file.getMetadata(), StandardCharsets.UTF_8);
		if (TextUtils.isEmpty(filePath) \|\| filePath.startsWith(STDIN_PATH)) {
		Metadata metadata = Metadata.fromByteArray(file.getMetadata());
		if (metadata == null) {
		Slog.e(TAG, "Invalid metadata for file: " + file.getName());
		return false;
		}
		switch (metadata.getMode()) {
		case Metadata.STDIN: {
		final ParcelFileDescriptor inFd = getStdInPFD(shellCommand);
		mConnector.writeData(file.getName(), 0, file.getLengthBytes(), inFd);
		} else {
		break;
		}
		case Metadata.LOCAL_FILE: {
		ParcelFileDescriptor incomingFd = null;
		try {
		incomingFd = getLocalFile(shellCommand, filePath);
		incomingFd = getLocalFile(shellCommand, metadata.getData());
		mConnector.writeData(file.getName(), 0, incomingFd.getStatSize(),
		incomingFd);
		} finally {
		IoUtils.closeQuietly(incomingFd);
		}
		break;
		}
		default:
		Slog.e(TAG, "Unsupported metadata mode: " + metadata.getMode());
		return false;
		}
		}
		return true;

services/core/jni/com_android_server_pm_PackageManagerShellCommandDataLoader.cpp

+90 −49

Original line number	Diff line number	Diff line
		@@ -227,56 +227,40 @@ static inline unique_fd convertPfdToFdAndDup(JNIEnv* env, const JniIds& jni, job
		return result;
		}

		enum MetadataMode : int8_t {
		STDIN = 0,
		LOCAL_FILE = 1,
		DATA_ONLY_STREAMING = 2,
		STREAMING = 3,
		};

		struct InputDesc {
		unique_fd fd;
		IncFsSize size;
		IncFsBlockKind kind = INCFS_BLOCK_KIND_DATA;
		bool waitOnEof = false;
		bool streaming = false;
		MetadataMode mode = STDIN;
		};
		using InputDescs = std::vector<InputDesc>;

		static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
		IncFsSize size, IncFsSpan metadata) {
		InputDescs result;
		result.reserve(2);

		if (metadata.size == 0 \|\| *metadata.data == '-') {
		// stdin
		auto fd = convertPfdToFdAndDup(
		env, jni,
		env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
		jni.pmscdGetStdInPFD, shellCommand));
		if (fd.ok()) {
		result.push_back(InputDesc{
		.fd = std::move(fd),
		.size = size,
		.waitOnEof = true,
		});
		}
		return result;
		}
		if (*metadata.data == '+') {
		// verity tree from stdin, rest is streaming
		auto fd = convertPfdToFdAndDup(
		env, jni,
		env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
		jni.pmscdGetStdInPFD, shellCommand));
		if (fd.ok()) {
		auto treeSize = verityTreeSizeForFile(size);
		result.push_back(InputDesc{
		.fd = std::move(fd),
		.size = treeSize,
		.kind = INCFS_BLOCK_KIND_HASH,
		.waitOnEof = true,
		.streaming = true,
		});
		template <class T>
		std::optional<T> read(IncFsSpan& data) {
		if (data.size < (int32_t)sizeof(T)) {
		return {};
		}
		return result;
		T res;
		memcpy(&res, data.data, sizeof(res));
		data.data += sizeof(res);
		data.size -= sizeof(res);
		return res;
		}

		// local file and possibly signature
		const std::string filePath(metadata.data, metadata.size);
		static inline InputDescs openLocalFile(JNIEnv* env, const JniIds& jni, jobject shellCommand,
		IncFsSize size, const std::string& filePath) {
		InputDescs result;
		result.reserve(2);

		const std::string idsigPath = filePath + ".idsig";

		auto idsigFd = convertPfdToFdAndDup(
		@@ -314,6 +298,59 @@ static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shel
		return result;
		}

		static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
		IncFsSize size, IncFsSpan metadata) {
		auto mode = read<int8_t>(metadata).value_or(STDIN);
		if (mode == LOCAL_FILE) {
		// local file and possibly signature
		return openLocalFile(env, jni, shellCommand, size,
		std::string(metadata.data, metadata.size));
		}

		auto fd = convertPfdToFdAndDup(
		env, jni,
		env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
		jni.pmscdGetStdInPFD, shellCommand));
		if (!fd.ok()) {
		return {};
		}

		InputDescs result;
		switch (mode) {
		case STDIN: {
		result.push_back(InputDesc{
		.fd = std::move(fd),
		.size = size,
		.waitOnEof = true,
		});
		break;
		}
		case DATA_ONLY_STREAMING: {
		// verity tree from stdin, rest is streaming
		auto treeSize = verityTreeSizeForFile(size);
		result.push_back(InputDesc{
		.fd = std::move(fd),
		.size = treeSize,
		.kind = INCFS_BLOCK_KIND_HASH,
		.waitOnEof = true,
		.streaming = true,
		.mode = DATA_ONLY_STREAMING,
		});
		break;
		}
		case STREAMING: {
		result.push_back(InputDesc{
		.fd = std::move(fd),
		.size = 0,
		.streaming = true,
		.mode = STREAMING,
		});
		break;
		}
		}
		return result;
		}

		static inline JNIEnv* GetJNIEnvironment(JavaVM* vm) {
		JNIEnv* env;
		if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
		@@ -390,6 +427,7 @@ private:
		blocks.reserve(BLOCKS_COUNT);

		unique_fd streamingFd;
		MetadataMode streamingMode;
		for (auto&& file : addedFiles) {
		auto inputs = openInputs(env, jni, shellCommand, file.size, file.metadata);
		if (inputs.empty()) {
		@@ -411,6 +449,7 @@ private:
		for (auto&& input : inputs) {
		if (input.streaming && !streamingFd.ok()) {
		streamingFd.reset(dup(input.fd));
		streamingMode = input.mode;
		}
		if (!copyToIncFs(incfsFd, input.size, input.kind, input.fd, input.waitOnEof,
		&buffer, &blocks)) {
		@@ -425,7 +464,7 @@ private:

		if (streamingFd.ok()) {
		ALOGE("onPrepareImage: done, proceeding to streaming.");
		return initStreaming(std::move(streamingFd));
		return initStreaming(std::move(streamingFd), streamingMode);
		}

		ALOGE("onPrepareImage: done.");
		@@ -564,7 +603,7 @@ private:
		}

		// Streaming.
		bool initStreaming(unique_fd inout) {
		bool initStreaming(unique_fd inout, MetadataMode mode) {
		mEventFd.reset(eventfd(0, EFD_CLOEXEC));
		if (mEventFd < 0) {
		ALOGE("Failed to create eventfd.");
		@@ -591,8 +630,8 @@ private:
		}
		}

		mReceiverThread =
		std::thread([this, io = std::move(inout)]() mutable { receiver(std::move(io)); });
		mReceiverThread = std::thread(
		[this, io = std::move(inout), mode]() mutable { receiver(std::move(io), mode); });
		ALOGI("Started streaming...");
		return true;
		}
		@@ -624,7 +663,7 @@ private:
		}
		}

		void receiver(unique_fd inout) {
		void receiver(unique_fd inout, MetadataMode mode) {
		std::vector<uint8_t> data;
		std::vector<IncFsDataBlock> instructions;
		std::unordered_map<FileIdx, unique_fd> writeFds;
		@@ -667,7 +706,7 @@ private:
		break;
		}
		const FileIdx fileIdx = header.fileIdx;
		const android::dataloader::FileId fileId = convertFileIndexToFileId(fileIdx);
		const android::dataloader::FileId fileId = convertFileIndexToFileId(mode, fileIdx);
		if (!android::incfs::isValidFileId(fileId)) {
		ALOGE("Unknown data destination for file ID %d. "
		"Ignore.",
		@@ -679,7 +718,7 @@ private:
		if (writeFd < 0) {
		writeFd.reset(this->mIfs->openWrite(fileId));
		if (writeFd < 0) {
		ALOGE("Failed to open file %d for writing (%d). Aboring.", header.fileIdx,
		ALOGE("Failed to open file %d for writing (%d). Aborting.", header.fileIdx,
		-writeFd);
		break;
		}
		@@ -716,9 +755,11 @@ private:
		}

		FileIdx convertFileIdToFileIndex(android::dataloader::FileId fileId) {
		// FileId is a string in format '+FileIdx\0'.
		// FileId has format '\2FileIdx'.
		const char* meta = (const char*)&fileId;
		if (*meta != '+') {

		int8_t mode = *meta;
		if (mode != DATA_ONLY_STREAMING && mode != STREAMING) {
		return -1;
		}

		@@ -732,10 +773,10 @@ private:
		return FileIdx(fileIdx);
		}

		android::dataloader::FileId convertFileIndexToFileId(FileIdx fileIdx) {
		android::dataloader::FileId convertFileIndexToFileId(MetadataMode mode, FileIdx fileIdx) {
		IncFsFileId fileId = {};
		char* meta = (char*)&fileId;
		*meta = '+';
		*meta = mode;
		if (auto [p, ec] = std::to_chars(meta + 1, meta + sizeof(fileId), fileIdx);
		ec != std::errc()) {
		return {};