Merge "support for CEA-608 closed caption" (3ea91c95) · Commits · e / os / android_frameworks_av

include/media/stagefright/MediaDefs.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -61,6 +61,7 @@ extern const char *MEDIA_MIMETYPE_CONTAINER_WVM;
		extern const char *MEDIA_MIMETYPE_TEXT_3GPP;
		extern const char *MEDIA_MIMETYPE_TEXT_SUBRIP;
		extern const char *MEDIA_MIMETYPE_TEXT_VTT;
		extern const char *MEDIA_MIMETYPE_TEXT_CEA_608;

		} // namespace android

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+74 −3

Original line number	Diff line number	Diff line
		@@ -375,14 +375,24 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		inbandTracks = mSource->getTrackCount();
		}

		size_t ccTracks = 0;
		if (mCCDecoder != NULL) {
		ccTracks = mCCDecoder->getTrackCount();
		}

		// total track count
		reply->writeInt32(inbandTracks);
		reply->writeInt32(inbandTracks + ccTracks);

		// write inband tracks
		for (size_t i = 0; i < inbandTracks; ++i) {
		writeTrackInfo(reply, mSource->getTrackInfo(i));
		}

		// write CC track
		for (size_t i = 0; i < ccTracks; ++i) {
		writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
		}

		sp<AMessage> response = new AMessage;
		response->postReply(replyID);
		break;
		@@ -404,9 +414,19 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		if (mSource != NULL) {
		inbandTracks = mSource->getTrackCount();
		}
		size_t ccTracks = 0;
		if (mCCDecoder != NULL) {
		ccTracks = mCCDecoder->getTrackCount();
		}

		if (trackIndex < inbandTracks) {
		err = mSource->selectTrack(trackIndex, select);
		} else {
		trackIndex -= inbandTracks;

		if (trackIndex < ccTracks) {
		err = mCCDecoder->selectTrack(trackIndex, select);
		}
		}

		sp<AMessage> response = new AMessage;
		@@ -870,6 +890,12 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		break;
		}

		case kWhatClosedCaptionNotify:
		{
		onClosedCaptionNotify(msg);
		break;
		}

		default:
		TRESPASS();
		break;
		@@ -933,6 +959,9 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
		AString mime;
		CHECK(format->findString("mime", &mime));
		mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());

		sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
		mCCDecoder = new CCDecoder(ccNotify);
		}

		sp<AMessage> notify =
		@@ -1073,6 +1102,10 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
		mediaTimeUs / 1E6);
		#endif

		if (!audio) {
		mCCDecoder->decode(accessUnit);
		}

		reply->setBuffer("buffer", accessUnit);
		reply->post();

		@@ -1101,14 +1134,15 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
		sp<ABuffer> buffer;
		CHECK(msg->findBuffer("buffer", &buffer));

		int64_t mediaTimeUs;
		CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));

		int64_t &skipUntilMediaTimeUs =
		audio
		? mSkipRenderingAudioUntilMediaTimeUs
		: mSkipRenderingVideoUntilMediaTimeUs;

		if (skipUntilMediaTimeUs >= 0) {
		int64_t mediaTimeUs;
		CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));

		if (mediaTimeUs < skipUntilMediaTimeUs) {
		ALOGV("dropping %s buffer at time %lld as requested.",
		@@ -1122,6 +1156,10 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
		skipUntilMediaTimeUs = -1;
		}

		if (!audio && mCCDecoder->isSelected()) {
		mCCDecoder->display(mediaTimeUs);
		}

		mRenderer->queueBuffer(audio, buffer, reply);
		}

		@@ -1510,6 +1548,39 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
		}
		}

		void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
		int32_t what;
		CHECK(msg->findInt32("what", &what));

		switch (what) {
		case NuPlayer::CCDecoder::kWhatClosedCaptionData:
		{
		sp<ABuffer> buffer;
		CHECK(msg->findBuffer("buffer", &buffer));

		size_t inbandTracks = 0;
		if (mSource != NULL) {
		inbandTracks = mSource->getTrackCount();
		}

		sendSubtitleData(buffer, inbandTracks);
		break;
		}

		case NuPlayer::CCDecoder::kWhatTrackAdded:
		{
		notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);

		break;
		}

		default:
		TRESPASS();
		}


		}

		void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
		int32_t trackIndex;
		int64_t timeUs, durationUs;

media/libmediaplayerservice/nuplayer/NuPlayer.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -76,6 +76,7 @@ public:

		private:
		struct Decoder;
		struct CCDecoder;
		struct GenericSource;
		struct HTTPLiveSource;
		struct Renderer;
		@@ -98,6 +99,7 @@ private:
		kWhatScanSources = 'scan',
		kWhatVideoNotify = 'vidN',
		kWhatAudioNotify = 'audN',
		kWhatClosedCaptionNotify = 'capN',
		kWhatRendererNotify = 'renN',
		kWhatReset = 'rset',
		kWhatSeek = 'seek',
		@@ -119,6 +121,7 @@ private:
		sp<Decoder> mVideoDecoder;
		bool mVideoIsAVC;
		sp<Decoder> mAudioDecoder;
		sp<CCDecoder> mCCDecoder;
		sp<Renderer> mRenderer;

		List<sp<Action> > mDeferredActions;
		@@ -186,6 +189,7 @@ private:
		void performSetSurface(const sp<NativeWindowWrapper> &wrapper);

		void onSourceNotify(const sp<AMessage> &msg);
		void onClosedCaptionNotify(const sp<AMessage> &msg);

		void queueDecoderShutdown(
		bool audio, bool video, const sp<AMessage> &reply);

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp

+268 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@
		#include "NuPlayerDecoder.h"

		#include <media/ICrypto.h>
		#include <media/stagefright/foundation/ABitReader.h>
		#include <media/stagefright/foundation/ABuffer.h>
		#include <media/stagefright/foundation/ADebug.h>
		#include <media/stagefright/foundation/AMessage.h>
		@@ -535,5 +536,272 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetF
		return seamless;
		}

		struct NuPlayer::CCDecoder::CCData {
		CCData(uint8_t type, uint8_t data1, uint8_t data2)
		: mType(type), mData1(data1), mData2(data2) {
		}

		uint8_t mType;
		uint8_t mData1;
		uint8_t mData2;
		};

		NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
		: mNotify(notify),
		mTrackCount(0),
		mSelectedTrack(-1) {
		}

		size_t NuPlayer::CCDecoder::getTrackCount() const {
		return mTrackCount;
		}

		sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
		CHECK(index == 0);

		sp<AMessage> format = new AMessage();

		format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
		format->setString("language", "und");
		format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
		format->setInt32("auto", 1);
		format->setInt32("default", 1);
		format->setInt32("forced", 0);

		return format;
		}

		status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
		CHECK(index < mTrackCount);

		if (select) {
		if (mSelectedTrack == (ssize_t)index) {
		ALOGE("track %zu already selected", index);
		return BAD_VALUE;
		}
		ALOGV("selected track %zu", index);
		mSelectedTrack = index;
		} else {
		if (mSelectedTrack != (ssize_t)index) {
		ALOGE("track %zu is not selected", index);
		return BAD_VALUE;
		}
		ALOGV("unselected track %zu", index);
		mSelectedTrack = -1;
		}

		return OK;
		}

		bool NuPlayer::CCDecoder::isSelected() const {
		return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)mTrackCount;
		}

		bool NuPlayer::CCDecoder::isNullPad(CCData *cc) const {
		return cc->mData1 < 0x10 && cc->mData2 < 0x10;
		}

		void NuPlayer::CCDecoder::dumpBytePair(const sp<ABuffer> &ccBuf) const {
		size_t offset = 0;
		AString out;

		while (offset < ccBuf->size()) {
		char tmp[128];

		CCData cc = (CCData ) (ccBuf->data() + offset);

		if (isNullPad(cc)) {
		// 1 null pad or XDS metadata, ignore
		offset += sizeof(CCData);
		continue;
		}

		if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
		// 2 basic chars
		sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
		} else if ((cc->mData1 == 0x11 \|\| cc->mData1 == 0x19)
		&& cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
		// 1 special char
		sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else if ((cc->mData1 == 0x12 \|\| cc->mData1 == 0x1A)
		&& cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
		// 1 Spanish/French char
		sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else if ((cc->mData1 == 0x13 \|\| cc->mData1 == 0x1B)
		&& cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
		// 1 Portuguese/German/Danish char
		sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else if ((cc->mData1 == 0x11 \|\| cc->mData1 == 0x19)
		&& cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
		// Mid-Row Codes (Table 69)
		sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else if (((cc->mData1 == 0x14 \|\| cc->mData1 == 0x1c)
		&& cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
		\|\|
		((cc->mData1 == 0x17 \|\| cc->mData1 == 0x1f)
		&& cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
		// Misc Control Codes (Table 70)
		sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else if ((cc->mData1 & 0x70) == 0x10
		&& (cc->mData2 & 0x40) == 0x40
		&& ((cc->mData1 & 0x07) \|\| !(cc->mData2 & 0x20)) ) {
		// Preamble Address Codes (Table 71)
		sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		} else {
		sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
		}

		if (out.size() > 0) {
		out.append(", ");
		}

		out.append(tmp);

		offset += sizeof(CCData);
		}

		ALOGI("%s", out.c_str());
		}

		bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
		int64_t timeUs;
		CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));

		sp<ABuffer> sei;
		if (!accessUnit->meta()->findBuffer("sei", &sei) \|\| sei == NULL) {
		return false;
		}

		bool hasCC = false;

		ABitReader br(sei->data() + 1, sei->size() - 1);
		// sei_message()
		while (br.numBitsLeft() >= 16) { // at least 16-bit for sei_message()
		uint32_t payload_type = 0;
		size_t payload_size = 0;
		uint8_t last_byte;

		do {
		last_byte = br.getBits(8);
		payload_type += last_byte;
		} while (last_byte == 0xFF);

		do {
		last_byte = br.getBits(8);
		payload_size += last_byte;
		} while (last_byte == 0xFF);

		// sei_payload()
		if (payload_type == 4) {
		// user_data_registered_itu_t_t35()

		// ATSC A/72: 6.4.2
		uint8_t itu_t_t35_country_code = br.getBits(8);
		uint16_t itu_t_t35_provider_code = br.getBits(16);
		uint32_t user_identifier = br.getBits(32);
		uint8_t user_data_type_code = br.getBits(8);

		payload_size -= 1 + 2 + 4 + 1;

		if (itu_t_t35_country_code == 0xB5
		&& itu_t_t35_provider_code == 0x0031
		&& user_identifier == 'GA94'
		&& user_data_type_code == 0x3) {
		hasCC = true;

		// MPEG_cc_data()
		// ATSC A/53 Part 4: 6.2.3.1
		br.skipBits(1); //process_em_data_flag
		bool process_cc_data_flag = br.getBits(1);
		br.skipBits(1); //additional_data_flag
		size_t cc_count = br.getBits(5);
		br.skipBits(8); // em_data;
		payload_size -= 2;

		if (process_cc_data_flag) {
		AString out;

		sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
		ccBuf->setRange(0, 0);

		for (size_t i = 0; i < cc_count; i++) {
		uint8_t marker = br.getBits(5);
		CHECK_EQ(marker, 0x1f);

		bool cc_valid = br.getBits(1);
		uint8_t cc_type = br.getBits(2);
		// remove odd parity bit
		uint8_t cc_data_1 = br.getBits(8) & 0x7f;
		uint8_t cc_data_2 = br.getBits(8) & 0x7f;

		if (cc_valid
		&& (cc_type == 0 \|\| cc_type == 1)) {
		CCData cc(cc_type, cc_data_1, cc_data_2);
		if (!isNullPad(&cc)) {
		memcpy(ccBuf->data() + ccBuf->size(),
		(void *)&cc, sizeof(cc));
		ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
		}
		}
		}
		payload_size -= cc_count * 3;

		mCCMap.add(timeUs, ccBuf);
		break;
		}
		} else {
		ALOGV("Malformed SEI payload type 4");
		}
		} else {
		ALOGV("Unsupported SEI payload type %d", payload_type);
		}

		// skipping remaining bits of this payload
		br.skipBits(payload_size * 8);
		}

		return hasCC;
		}

		void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
		if (extractFromSEI(accessUnit) && mTrackCount == 0) {
		mTrackCount++;

		ALOGI("Found CEA-608 track");
		sp<AMessage> msg = mNotify->dup();
		msg->setInt32("what", kWhatTrackAdded);
		msg->post();
		}
		// TODO: extract CC from other sources
		}

		void NuPlayer::CCDecoder::display(int64_t timeUs) {
		ssize_t index = mCCMap.indexOfKey(timeUs);
		if (index < 0) {
		ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
		return;
		}

		sp<ABuffer> &ccBuf = mCCMap.editValueAt(index);

		if (ccBuf->size() > 0) {
		#if 0
		dumpBytePair(ccBuf);
		#endif

		ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
		ccBuf->meta()->setInt64("timeUs", timeUs);
		ccBuf->meta()->setInt64("durationUs", 0ll);

		sp<AMessage> msg = mNotify->dup();
		msg->setInt32("what", kWhatClosedCaptionData);
		msg->setBuffer("buffer", ccBuf);
		msg->post();
		}

		// remove all entries before timeUs
		mCCMap.removeItemsAt(0, index + 1);
		}

		} // namespace android

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h

+30 −0

Original line number	Diff line number	Diff line
		@@ -101,6 +101,36 @@ private:
		DISALLOW_EVIL_CONSTRUCTORS(Decoder);
		};

		struct NuPlayer::CCDecoder : public RefBase {
		enum {
		kWhatClosedCaptionData,
		kWhatTrackAdded,
		};

		CCDecoder(const sp<AMessage> &notify);

		size_t getTrackCount() const;
		sp<AMessage> getTrackInfo(size_t index) const;
		status_t selectTrack(size_t index, bool select);
		bool isSelected() const;
		void decode(const sp<ABuffer> &accessUnit);
		void display(int64_t timeUs);

		private:
		struct CCData;

		sp<AMessage> mNotify;
		KeyedVector<int64_t, sp<ABuffer> > mCCMap;
		size_t mTrackCount;
		int32_t mSelectedTrack;

		bool isNullPad(CCData *cc) const;
		void dumpBytePair(const sp<ABuffer> &ccBuf) const;
		bool extractFromSEI(const sp<ABuffer> &accessUnit);

		DISALLOW_EVIL_CONSTRUCTORS(CCDecoder);
		};

		} // namespace android

		#endif // NUPLAYER_DECODER_H_