Merge "Add MediaMetrics support to MediaParser" (49818960) · Commits · e / os / android_frameworks_base

apex/media/framework/Android.bp

+24 −2

Original line number	Diff line number	Diff line
		@@ -35,7 +35,6 @@ java_library {
		libs: [
		"framework_media_annotation",
		],

		static_libs: [
		"exoplayer2-extractor"
		],
		@@ -111,10 +110,33 @@ java_sdk_library {
		impl_library_visibility: ["//frameworks/av/apex:__subpackages__"],
		}


		java_library {
		name: "framework_media_annotation",
		srcs: [":framework-media-annotation-srcs"],
		installable: false,
		sdk_version: "core_current",
		}

		cc_library_shared {
		name: "libmediaparser-jni",
		srcs: [
		"jni/android_media_MediaParserJNI.cpp",
		],
		header_libs: ["jni_headers"],
		shared_libs: [
		"libandroid",
		"liblog",
		"libmediametrics",
		],
		cflags: [
		"-Wall",
		"-Werror",
		"-Wno-unused-parameter",
		"-Wunreachable-code",
		"-Wunused",
		],
		apex_available: [
		"com.android.media",
		],
		min_sdk_version: "29",
		}

apex/media/framework/java/android/media/MediaParser.java

+149 −11

Original line number	Diff line number	Diff line
		@@ -75,6 +75,8 @@ import java.util.LinkedHashMap;
		import java.util.List;
		import java.util.Map;
		import java.util.UUID;
		import java.util.concurrent.ThreadLocalRandom;
		import java.util.function.Function;

		/**
		* Parses media container formats and extracts contained media samples and metadata.
		@@ -882,6 +884,7 @@ public final class MediaParser {
		// Private constants.

		private static final String TAG = "MediaParser";
		private static final String JNI_LIBRARY_NAME = "mediaparser-jni";
		private static final Map<String, ExtractorFactory> EXTRACTOR_FACTORIES_BY_NAME;
		private static final Map<String, Class> EXPECTED_TYPE_BY_PARAMETER_NAME;
		private static final String TS_MODE_SINGLE_PMT = "single_pmt";
		@@ -889,6 +892,14 @@ public final class MediaParser {
		private static final String TS_MODE_HLS = "hls";
		private static final int BYTES_PER_SUBSAMPLE_ENCRYPTION_ENTRY = 6;
		private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
		private static final String MEDIAMETRICS_ELEMENT_SEPARATOR = "\|";
		private static final int MEDIAMETRICS_MAX_STRING_SIZE = 200;
		private static final int MEDIAMETRICS_PARAMETER_LIST_MAX_LENGTH;
		/**
		* Intentional error introduced to reported metrics to prevent identification of the parsed
		* media. Note: Increasing this value may cause older hostside CTS tests to fail.
		*/
		private static final float MEDIAMETRICS_DITHER = .02f;

		@IntDef(
		value = {
		@@ -920,7 +931,7 @@ public final class MediaParser {
		@NonNull @ParserName String name, @NonNull OutputConsumer outputConsumer) {
		String[] nameAsArray = new String[] {name};
		assertValidNames(nameAsArray);
		return new MediaParser(outputConsumer, /* sniff= */ false, name);
		return new MediaParser(outputConsumer, /* createdByName= */ true, name);
		}

		/**
		@@ -940,7 +951,7 @@ public final class MediaParser {
		if (parserNames.length == 0) {
		parserNames = EXTRACTOR_FACTORIES_BY_NAME.keySet().toArray(new String[0]);
		}
		return new MediaParser(outputConsumer, /* sniff= */ true, parserNames);
		return new MediaParser(outputConsumer, /* createdByName= */ false, parserNames);
		}

		// Misc static methods.
		@@ -1052,6 +1063,14 @@ public final class MediaParser {
		private long mPendingSeekPosition;
		private long mPendingSeekTimeMicros;
		private boolean mLoggedSchemeInitDataCreationException;
		private boolean mReleased;

		// MediaMetrics fields.
		private final boolean mCreatedByName;
		private final SparseArray<Format> mTrackFormats;
		private String mLastObservedExceptionName;
		private long mDurationMillis;
		private long mResourceByteCount;

		// Public methods.

		@@ -1166,11 +1185,16 @@ public final class MediaParser {
		if (mExtractorInput == null) {
		// TODO: For efficiency, the same implementation should be used, by providing a
		// clearBuffers() method, or similar.
		long resourceLength = seekableInputReader.getLength();
		if (resourceLength == -1) {
		mResourceByteCount = -1;
		}
		if (mResourceByteCount != -1) {
		mResourceByteCount += resourceLength;
		}
		mExtractorInput =
		new DefaultExtractorInput(
		mExoDataReader,
		seekableInputReader.getPosition(),
		seekableInputReader.getLength());
		mExoDataReader, seekableInputReader.getPosition(), resourceLength);
		}
		mExoDataReader.mInputReader = seekableInputReader;

		@@ -1195,7 +1219,10 @@ public final class MediaParser {
		}
		}
		if (mExtractor == null) {
		throw UnrecognizedInputFormatException.createForExtractors(mParserNamesPool);
		UnrecognizedInputFormatException exception =
		UnrecognizedInputFormatException.createForExtractors(mParserNamesPool);
		mLastObservedExceptionName = exception.getClass().getName();
		throw exception;
		}
		return true;
		}
		@@ -1223,8 +1250,13 @@ public final class MediaParser {
		int result;
		try {
		result = mExtractor.read(mExtractorInput, mPositionHolder);
		} catch (ParserException e) {
		throw new ParsingException(e);
		} catch (Exception e) {
		mLastObservedExceptionName = e.getClass().getName();
		if (e instanceof ParserException) {
		throw new ParsingException((ParserException) e);
		} else {
		throw e;
		}
		}
		if (result == Extractor.RESULT_END_OF_INPUT) {
		mExtractorInput = null;
		@@ -1264,21 +1296,64 @@ public final class MediaParser {
		* invoked.
		*/
		public void release() {
		// TODO: Dump media metrics here.
		mExtractorInput = null;
		mExtractor = null;
		if (mReleased) {
		// Nothing to do.
		return;
		}
		mReleased = true;

		String trackMimeTypes = buildMediaMetricsString(format -> format.sampleMimeType);
		String trackCodecs = buildMediaMetricsString(format -> format.codecs);
		int videoWidth = -1;
		int videoHeight = -1;
		for (int i = 0; i < mTrackFormats.size(); i++) {
		Format format = mTrackFormats.valueAt(i);
		if (format.width != Format.NO_VALUE && format.height != Format.NO_VALUE) {
		videoWidth = format.width;
		videoHeight = format.height;
		break;
		}
		}

		String alteredParameters =
		String.join(
		MEDIAMETRICS_ELEMENT_SEPARATOR,
		mParserParameters.keySet().toArray(new String[0]));
		alteredParameters =
		alteredParameters.substring(
		0,
		Math.min(
		alteredParameters.length(),
		MEDIAMETRICS_PARAMETER_LIST_MAX_LENGTH));

		nativeSubmitMetrics(
		mParserName,
		mCreatedByName,
		String.join(MEDIAMETRICS_ELEMENT_SEPARATOR, mParserNamesPool),
		mLastObservedExceptionName,
		addDither(mResourceByteCount),
		addDither(mDurationMillis),
		trackMimeTypes,
		trackCodecs,
		alteredParameters,
		videoWidth,
		videoHeight);
		}

		// Private methods.

		private MediaParser(OutputConsumer outputConsumer, boolean sniff, String... parserNamesPool) {
		private MediaParser(
		OutputConsumer outputConsumer, boolean createdByName, String... parserNamesPool) {
		if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
		throw new UnsupportedOperationException("Android version must be R or greater.");
		}
		mParserParameters = new HashMap<>();
		mOutputConsumer = outputConsumer;
		mParserNamesPool = parserNamesPool;
		mParserName = sniff ? PARSER_NAME_UNKNOWN : parserNamesPool[0];
		mCreatedByName = createdByName;
		mParserName = createdByName ? parserNamesPool[0] : PARSER_NAME_UNKNOWN;
		mPositionHolder = new PositionHolder();
		mExoDataReader = new InputReadingDataReader();
		removePendingSeek();
		@@ -1286,6 +1361,24 @@ public final class MediaParser {
		mScratchParsableByteArrayAdapter = new ParsableByteArrayAdapter();
		mSchemeInitDataConstructor = getSchemeInitDataConstructor();
		mMuxedCaptionFormats = new ArrayList<>();

		// MediaMetrics.
		mTrackFormats = new SparseArray<>();
		mLastObservedExceptionName = "";
		mDurationMillis = -1;
		}

		private String buildMediaMetricsString(Function<Format, String> formatFieldGetter) {
		StringBuilder stringBuilder = new StringBuilder();
		for (int i = 0; i < mTrackFormats.size(); i++) {
		if (i > 0) {
		stringBuilder.append(MEDIAMETRICS_ELEMENT_SEPARATOR);
		}
		String fieldValue = formatFieldGetter.apply(mTrackFormats.valueAt(i));
		stringBuilder.append(fieldValue != null ? fieldValue : "");
		}
		return stringBuilder.substring(
		0, Math.min(stringBuilder.length(), MEDIAMETRICS_MAX_STRING_SIZE));
		}

		private void setMuxedCaptionFormats(List<MediaFormat> mediaFormats) {
		@@ -1528,6 +1621,10 @@ public final class MediaParser {

		@Override
		public void seekMap(com.google.android.exoplayer2.extractor.SeekMap exoplayerSeekMap) {
		long durationUs = exoplayerSeekMap.getDurationUs();
		if (durationUs != C.TIME_UNSET) {
		mDurationMillis = C.usToMs(durationUs);
		}
		if (mExposeChunkIndexAsMediaFormat && exoplayerSeekMap instanceof ChunkIndex) {
		ChunkIndex chunkIndex = (ChunkIndex) exoplayerSeekMap;
		MediaFormat mediaFormat = new MediaFormat();
		@@ -1575,6 +1672,7 @@ public final class MediaParser {

		@Override
		public void format(Format format) {
		mTrackFormats.put(mTrackIndex, format);
		mOutputConsumer.onTrackDataFound(
		mTrackIndex,
		new TrackData(
		@@ -2031,6 +2129,20 @@ public final class MediaParser {
		return new SeekPoint(exoPlayerSeekPoint.timeUs, exoPlayerSeekPoint.position);
		}

		/**
		* Introduces random error to the given metric value in order to prevent the identification of
		* the parsed media.
		*/
		private static long addDither(long value) {
		// Generate a random in [0, 1].
		double randomDither = ThreadLocalRandom.current().nextFloat();
		// Clamp the random number to [0, 2 * MEDIAMETRICS_DITHER].
		randomDither = 2 MEDIAMETRICS_DITHER;
		// Translate the random number to [1 - MEDIAMETRICS_DITHER, 1 + MEDIAMETRICS_DITHER].
		randomDither += 1 - MEDIAMETRICS_DITHER;
		return value != -1 ? (long) (value * randomDither) : -1;
		}

		private static void assertValidNames(@NonNull String[] names) {
		for (String name : names) {
		if (!EXTRACTOR_FACTORIES_BY_NAME.containsKey(name)) {
		@@ -2070,9 +2182,26 @@ public final class MediaParser {
		}
		}

		// Native methods.

		private native void nativeSubmitMetrics(
		String parserName,
		boolean createdByName,
		String parserPool,
		String lastObservedExceptionName,
		long resourceByteCount,
		long durationMillis,
		String trackMimeTypes,
		String trackCodecs,
		String alteredParameters,
		int videoWidth,
		int videoHeight);

		// Static initialization.

		static {
		System.loadLibrary(JNI_LIBRARY_NAME);

		// Using a LinkedHashMap to keep the insertion order when iterating over the keys.
		LinkedHashMap<String, ExtractorFactory> extractorFactoriesByName = new LinkedHashMap<>();
		// Parsers are ordered to match ExoPlayer's DefaultExtractorsFactory extractor ordering,
		@@ -2125,6 +2254,15 @@ public final class MediaParser {
		// We do not check PARAMETER_EXPOSE_CAPTION_FORMATS here, and we do it in setParameters
		// instead. Checking that the value is a List is insufficient to catch wrong parameter
		// value types.
		int sumOfParameterNameLengths =
		expectedTypeByParameterName.keySet().stream()
		.map(String::length)
		.reduce(0, Integer::sum);
		sumOfParameterNameLengths += PARAMETER_EXPOSE_CAPTION_FORMATS.length();
		// Add space for any required separators.
		MEDIAMETRICS_PARAMETER_LIST_MAX_LENGTH =
		sumOfParameterNameLengths + expectedTypeByParameterName.size();

		EXPECTED_TYPE_BY_PARAMETER_NAME = Collections.unmodifiableMap(expectedTypeByParameterName);
		}
		}

apex/media/framework/jni/android_media_MediaParserJNI.cpp

0 → 100644

+92 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2020, The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include <jni.h>
		#include <media/MediaMetrics.h>

		#define JNI_FUNCTION(RETURN_TYPE, NAME, ...) \
		extern "C" { \
		JNIEXPORT RETURN_TYPE Java_android_media_MediaParser_##NAME(JNIEnv* env, jobject thiz, \
		##__VA_ARGS__); \
		} \
		JNIEXPORT RETURN_TYPE Java_android_media_MediaParser_##NAME(JNIEnv* env, jobject thiz, \
		##__VA_ARGS__)

		namespace {

		constexpr char kMediaMetricsKey[] = "mediaparser";

		constexpr char kAttributeParserName[] = "android.media.mediaparser.parserName";
		constexpr char kAttributeCreatedByName[] = "android.media.mediaparser.createdByName";
		constexpr char kAttributeParserPool[] = "android.media.mediaparser.parserPool";
		constexpr char kAttributeLastException[] = "android.media.mediaparser.lastException";
		constexpr char kAttributeResourceByteCount[] = "android.media.mediaparser.resourceByteCount";
		constexpr char kAttributeDurationMillis[] = "android.media.mediaparser.durationMillis";
		constexpr char kAttributeTrackMimeTypes[] = "android.media.mediaparser.trackMimeTypes";
		constexpr char kAttributeTrackCodecs[] = "android.media.mediaparser.trackCodecs";
		constexpr char kAttributeAlteredParameters[] = "android.media.mediaparser.alteredParameters";
		constexpr char kAttributeVideoWidth[] = "android.media.mediaparser.videoWidth";
		constexpr char kAttributeVideoHeight[] = "android.media.mediaparser.videoHeight";

		// Util class to handle string resource management.
		class JstringHandle {
		public:
		JstringHandle(JNIEnv* env, jstring value) : mEnv(env), mJstringValue(value) {
		mCstringValue = env->GetStringUTFChars(value, /* isCopy= */ nullptr);
		}

		~JstringHandle() {
		if (mCstringValue != nullptr) {
		mEnv->ReleaseStringUTFChars(mJstringValue, mCstringValue);
		}
		}

		[[nodiscard]] const char* value() const {
		return mCstringValue != nullptr ? mCstringValue : "";
		}

		JNIEnv* mEnv;
		jstring mJstringValue;
		const char* mCstringValue;
		};

		} // namespace

		JNI_FUNCTION(void, nativeSubmitMetrics, jstring parserNameJstring, jboolean createdByName,
		jstring parserPoolJstring, jstring lastExceptionJstring, jlong resourceByteCount,
		jlong durationMillis, jstring trackMimeTypesJstring, jstring trackCodecsJstring,
		jstring alteredParameters, jint videoWidth, jint videoHeight) {
		mediametrics_handle_t item(mediametrics_create(kMediaMetricsKey));
		mediametrics_setCString(item, kAttributeParserName,
		JstringHandle(env, parserNameJstring).value());
		mediametrics_setInt32(item, kAttributeCreatedByName, createdByName ? 1 : 0);
		mediametrics_setCString(item, kAttributeParserPool,
		JstringHandle(env, parserPoolJstring).value());
		mediametrics_setCString(item, kAttributeLastException,
		JstringHandle(env, lastExceptionJstring).value());
		mediametrics_setInt64(item, kAttributeResourceByteCount, resourceByteCount);
		mediametrics_setInt64(item, kAttributeDurationMillis, durationMillis);
		mediametrics_setCString(item, kAttributeTrackMimeTypes,
		JstringHandle(env, trackMimeTypesJstring).value());
		mediametrics_setCString(item, kAttributeTrackCodecs,
		JstringHandle(env, trackCodecsJstring).value());
		mediametrics_setCString(item, kAttributeAlteredParameters,
		JstringHandle(env, alteredParameters).value());
		mediametrics_setInt32(item, kAttributeVideoWidth, videoWidth);
		mediametrics_setInt32(item, kAttributeVideoHeight, videoHeight);
		mediametrics_selfRecord(item);
		mediametrics_delete(item);
		}

cmds/statsd/src/atoms.proto

+68 −1

Original line number	Diff line number	Diff line
		@@ -498,6 +498,7 @@ message Atom {
		CarrierIdMismatchEvent carrier_id_mismatch_event = 313 [(module) = "telephony"];
		CarrierIdMatchingTable carrier_id_table_update = 314 [(module) = "telephony"];
		DataStallRecoveryReported data_stall_recovery_reported = 315 [(module) = "telephony"];
		MediametricsMediaParserReported mediametrics_mediaparser_reported = 316;

		// StatsdStats tracks platform atoms with ids upto 500.
		// Update StatsdStats::kMaxPushedAtomId when atom ids here approach that value.
		@@ -8196,6 +8197,72 @@ message MediametricsExtractorReported {
		optional android.stats.mediametrics.ExtractorData extractor_data = 5 [(android.os.statsd.log_mode) = MODE_BYTES];
		}

		/**
		* Track MediaParser (parsing video/audio streams from containers) usage
		* Logged from:
		*
		* frameworks/av/services/mediametrics/statsd_mediaparser.cpp
		* frameworks/base/apex/media/framework/jni/android_media_MediaParserJNI.cpp
		*/
		message MediametricsMediaParserReported {
		optional int64 timestamp_nanos = 1;
		optional string package_name = 2;
		optional int64 package_version_code = 3;

		// MediaParser specific data.
		/**
		* The name of the parser selected for parsing the media, or an empty string
		* if no parser was selected.
		*/
		optional string parser_name = 4;
		/**
		* Whether the parser was created by name. 1 represents true, and 0
		* represents false.
		*/
		optional int32 created_by_name = 5;
		/**
		* The parser names in the sniffing pool separated by "\|".
		*/
		optional string parser_pool = 6;
		/**
		* The fully qualified name of the last encountered exception, or an empty
		* string if no exception was encountered.
		*/
		optional string last_exception = 7;
		/**
		* The size of the parsed media in bytes, or -1 if unknown. Note this value
		* contains intentional random error to prevent media content
		* identification.
		*/
		optional int64 resource_byte_count = 8;
		/**
		* The duration of the media in milliseconds, or -1 if unknown. Note this
		* value contains intentional random error to prevent media content
		* identification.
		*/
		optional int64 duration_millis = 9;
		/**
		* The MIME types of the tracks separated by "\|".
		*/
		optional string track_mime_types = 10;
		/**
		* The tracks' RFC 6381 codec strings separated by "\|".
		*/
		optional string track_codecs = 11;
		/**
		* Concatenation of the parameters altered by the client, separated by "\|".
		*/
		optional string altered_parameters = 12;
		/**
		* The video width in pixels, or -1 if unknown or not applicable.
		*/
		optional int32 video_width = 13;
		/**
		* The video height in pixels, or -1 if unknown or not applicable.
		*/
		optional int32 video_height = 14;
		}

		/**
		* Track how we arbitrate between microphone/input requests.
		* Logged from