[Rlog27] Add replay capability (f3731188) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/LatinIME.java

+1 −1

Original line number	Diff line number	Diff line
		@@ -428,7 +428,7 @@ public final class LatinIME extends InputMethodService implements KeyboardAction
		initSuggest();

		if (ProductionFlag.IS_EXPERIMENTAL) {
		ResearchLogger.getInstance().init(this);
		ResearchLogger.getInstance().init(this, mKeyboardSwitcher);
		}
		mDisplayOrientation = getResources().getConfiguration().orientation;

java/src/com/android/inputmethod/research/LogStatement.java

0 → 100644

+146 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2013 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.inputmethod.research;

		/**
		* A template for typed information stored in the logs.
		*
		* A LogStatement contains a name, keys, and flags about whether the {@code Object[] values}
		* associated with the {@code String[] keys} are likely to reveal information about the user. The
		* actual values are stored separately.
		*/
		class LogStatement {
		// Constants for particular statements
		public static final String TYPE_POINTER_TRACKER_CALL_LISTENER_ON_CODE_INPUT =
		"PointerTrackerCallListenerOnCodeInput";
		public static final String KEY_CODE = "code";
		public static final String VALUE_RESEARCH = "research";
		public static final String TYPE_LATIN_KEYBOARD_VIEW_ON_LONG_PRESS =
		"LatinKeyboardViewOnLongPress";
		public static final String ACTION = "action";
		public static final String VALUE_DOWN = "DOWN";
		public static final String TYPE_LATIN_KEYBOARD_VIEW_PROCESS_MOTION_EVENTS =
		"LatinKeyboardViewProcessMotionEvents";
		public static final String KEY_LOGGING_RELATED = "loggingRelated";

		// Name specifying the LogStatement type.
		private final String mType;

		// mIsPotentiallyPrivate indicates that event contains potentially private information. If
		// the word that this event is a part of is determined to be privacy-sensitive, then this
		// event should not be included in the output log. The system waits to output until the
		// containing word is known.
		private final boolean mIsPotentiallyPrivate;

		// mIsPotentiallyRevealing indicates that this statement may disclose details about other
		// words typed in other LogUnits. This can happen if the user is not inserting spaces, and
		// data from Suggestions and/or Composing text reveals the entire "megaword". For example,
		// say the user is typing "for the win", and the system wants to record the bigram "the
		// win". If the user types "forthe", omitting the space, the system will give "for the" as
		// a suggestion. If the user accepts the autocorrection, the suggestion for "for the" is
		// included in the log for the word "the", disclosing that the previous word had been "for".
		// For now, we simply do not include this data when logging part of a "megaword".
		private final boolean mIsPotentiallyRevealing;

		// mKeys stores the names that are the attributes in the output json objects
		private final String[] mKeys;
		private static final String[] NULL_KEYS = new String[0];

		LogStatement(final String name, final boolean isPotentiallyPrivate,
		final boolean isPotentiallyRevealing, final String... keys) {
		mType = name;
		mIsPotentiallyPrivate = isPotentiallyPrivate;
		mIsPotentiallyRevealing = isPotentiallyRevealing;
		mKeys = (keys == null) ? NULL_KEYS : keys;
		}

		public String getType() {
		return mType;
		}

		public boolean isPotentiallyPrivate() {
		return mIsPotentiallyPrivate;
		}

		public boolean isPotentiallyRevealing() {
		return mIsPotentiallyRevealing;
		}

		public String[] getKeys() {
		return mKeys;
		}

		/**
		* Utility function to test whether a key-value pair exists in a LogStatement.
		*
		* A LogStatement is really just a template -- it does not contain the values, only the
		* keys. So the values must be passed in as an argument.
		*
		* @param queryKey the String that is tested by {@code String.equals()} to the keys in the
		* LogStatement
		* @param queryValue an Object that must be {@code Object.equals()} to the key's corresponding
		* value in the {@code values} array
		* @param values the values corresponding to mKeys
		*
		* @returns {@true} if {@code queryKey} exists in the keys for this LogStatement, and {@code
		* queryValue} matches the corresponding value in {@code values}
		*
		* @throws IllegalArgumentException if {@code values.length} is not equal to keys().length()
		*/
		public boolean containsKeyValuePair(final String queryKey, final Object queryValue,
		final Object[] values) {
		if (mKeys.length != values.length) {
		throw new IllegalArgumentException("Mismatched number of keys and values.");
		}
		final int length = mKeys.length;
		for (int i = 0; i < length; i++) {
		if (mKeys[i].equals(queryKey) && values[i].equals(queryValue)) {
		return true;
		}
		}
		return false;
		}

		/**
		* Utility function to set a value in a LogStatement.
		*
		* A LogStatement is really just a template -- it does not contain the values, only the
		* keys. So the values must be passed in as an argument.
		*
		* @param queryKey the String that is tested by {@code String.equals()} to the keys in the
		* LogStatement
		* @param values the array of values corresponding to mKeys
		* @param newValue the replacement value to go into the {@code values} array
		*
		* @returns {@true} if the key exists and the value was successfully set, {@false} otherwise
		*
		* @throws IllegalArgumentException if {@code values.length} is not equal to keys().length()
		*/
		public boolean setValue(final String queryKey, final Object[] values, final Object newValue) {
		if (mKeys.length != values.length) {
		throw new IllegalArgumentException("Mismatched number of keys and values.");
		}
		final int length = mKeys.length;
		for (int i = 0; i < length; i++) {
		if (mKeys[i].equals(queryKey)) {
		values[i] = newValue;
		return true;
		}
		}
		return false;
		}
		}

java/src/com/android/inputmethod/research/LogUnit.java

+128 −11

Original line number	Diff line number	Diff line
		@@ -26,15 +26,12 @@ import android.view.inputmethod.CompletionInfo;
		import com.android.inputmethod.keyboard.Key;
		import com.android.inputmethod.latin.SuggestedWords;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.Utils;
		import com.android.inputmethod.latin.define.ProductionFlag;
		import com.android.inputmethod.research.ResearchLogger.LogStatement;

		import java.io.IOException;
		import java.io.StringWriter;
		import java.util.ArrayList;
		import java.util.List;
		import java.util.Map;

		/**
		* A group of log statements related to each other.
		@@ -53,6 +50,7 @@ import java.util.Map;
		/* package */ class LogUnit {
		private static final String TAG = LogUnit.class.getSimpleName();
		private static final boolean DEBUG = false && ProductionFlag.IS_EXPERIMENTAL_DEBUG;

		private final ArrayList<LogStatement> mLogStatementList;
		private final ArrayList<Object[]> mValuesList;
		// Assume that mTimeList is sorted in increasing order. Do not insert null values into
		@@ -142,10 +140,10 @@ import java.util.Map;
		JsonWriter jsonWriter = null;
		for (int i = 0; i < size; i++) {
		final LogStatement logStatement = mLogStatementList.get(i);
		if (!canIncludePrivateData && logStatement.mIsPotentiallyPrivate) {
		if (!canIncludePrivateData && logStatement.isPotentiallyPrivate()) {
		continue;
		}
		if (mIsPartOfMegaword && logStatement.mIsPotentiallyRevealing) {
		if (mIsPartOfMegaword && logStatement.isPotentiallyRevealing()) {
		continue;
		}
		// Only retrieve the jsonWriter if we need to. If we don't get this far, then
		@@ -228,16 +226,16 @@ import java.util.Map;
		private boolean outputLogStatementToLocked(final JsonWriter jsonWriter,
		final LogStatement logStatement, final Object[] values, final Long time) {
		if (DEBUG) {
		if (logStatement.mKeys.length != values.length) {
		Log.d(TAG, "Key and Value list sizes do not match. " + logStatement.mName);
		if (logStatement.getKeys().length != values.length) {
		Log.d(TAG, "Key and Value list sizes do not match. " + logStatement.getType());
		}
		}
		try {
		jsonWriter.beginObject();
		jsonWriter.name(CURRENT_TIME_KEY).value(System.currentTimeMillis());
		jsonWriter.name(UPTIME_KEY).value(time);
		jsonWriter.name(EVENT_TYPE_KEY).value(logStatement.mName);
		final String[] keys = logStatement.mKeys;
		jsonWriter.name(EVENT_TYPE_KEY).value(logStatement.getType());
		final String[] keys = logStatement.getKeys();
		final int length = values.length;
		for (int i = 0; i < length; i++) {
		jsonWriter.name(keys[i]);
		@@ -261,8 +259,8 @@ import java.util.Map;
		} else if (value == null) {
		jsonWriter.nullValue();
		} else {
		Log.w(TAG, "Unrecognized type to be logged: " +
		(value == null ? "<null>" : value.getClass().getName()));
		Log.w(TAG, "Unrecognized type to be logged: "
		+ (value == null ? "<null>" : value.getClass().getName()));
		jsonWriter.nullValue();
		}
		}
		@@ -422,4 +420,123 @@ import java.util.Map;
		}
		return false;
		}

		/**
		* Remove data associated with selecting the Research button.
		*
		* A LogUnit will capture all user interactions with the IME, including the "meta-interactions"
		* of using the Research button to control the logging (e.g. by starting and stopping recording
		* of a test case). Because meta-interactions should not be part of the normal log, calling
		* this method will set a field in the LogStatements of the motion events to indiciate that
		* they should be disregarded.
		*
		* This implementation assumes that the data recorded by the meta-interaction takes the
		* form of all events following the first MotionEvent.ACTION_DOWN before the first long-press
		* before the last onCodeEvent containing a code matching {@code LogStatement.VALUE_RESEARCH}.
		*
		* @returns true if data was removed
		*/
		public boolean removeResearchButtonInvocation() {
		// This method is designed to be idempotent.

		// First, find last invocation of "research" key
		final int indexOfLastResearchKey = findLastIndexContainingKeyValue(
		LogStatement.TYPE_POINTER_TRACKER_CALL_LISTENER_ON_CODE_INPUT,
		LogStatement.KEY_CODE, LogStatement.VALUE_RESEARCH);
		if (indexOfLastResearchKey < 0) {
		// Could not find invocation of "research" key. Leave log as is.
		if (DEBUG) {
		Log.d(TAG, "Could not find research key");
		}
		return false;
		}

		// Look for the long press that started the invocation of the research key code input.
		final int indexOfLastLongPressBeforeResearchKey =
		findLastIndexBefore(LogStatement.TYPE_LATIN_KEYBOARD_VIEW_ON_LONG_PRESS,
		indexOfLastResearchKey);

		// Look for DOWN event preceding the long press
		final int indexOfLastDownEventBeforeLongPress =
		findLastIndexContainingKeyValueBefore(
		LogStatement.TYPE_LATIN_KEYBOARD_VIEW_PROCESS_MOTION_EVENTS,
		LogStatement.ACTION, LogStatement.VALUE_DOWN,
		indexOfLastLongPressBeforeResearchKey);

		// Flag all LatinKeyboardViewProcessMotionEvents from the DOWN event to the research key as
		// logging-related
		final int startingIndex = indexOfLastDownEventBeforeLongPress == -1 ? 0
		: indexOfLastDownEventBeforeLongPress;
		for (int index = startingIndex; index < indexOfLastResearchKey; index++) {
		final LogStatement logStatement = mLogStatementList.get(index);
		final String type = logStatement.getType();
		final Object[] values = mValuesList.get(index);
		if (type.equals(LogStatement.TYPE_LATIN_KEYBOARD_VIEW_PROCESS_MOTION_EVENTS)) {
		logStatement.setValue(LogStatement.KEY_LOGGING_RELATED, values, true);
		}
		}
		return true;
		}

		/**
		* Find the index of the last LogStatement before {@code startingIndex} of type {@code type}.
		*
		* @param queryType a String that must be {@code String.equals()} to the LogStatement type
		* @param startingIndex the index to start the backward search from. Must be less than the
		* length of mLogStatementList, or an IndexOutOfBoundsException is thrown. Can be negative,
		* in which case -1 is returned.
		*
		* @return The index of the last LogStatement, -1 if none exists.
		*/
		private int findLastIndexBefore(final String queryType, final int startingIndex) {
		return findLastIndexContainingKeyValueBefore(queryType, null, null, startingIndex);
		}

		/**
		* Find the index of the last LogStatement before {@code startingIndex} of type {@code type}
		* containing the given key-value pair.
		*
		* @param queryType a String that must be {@code String.equals()} to the LogStatement type
		* @param queryKey a String that must be {@code String.equals()} to a key in the LogStatement
		* @param queryValue an Object that must be {@code String.equals()} to the key's corresponding
		* value
		*
		* @return The index of the last LogStatement, -1 if none exists.
		*/
		private int findLastIndexContainingKeyValue(final String queryType, final String queryKey,
		final Object queryValue) {
		return findLastIndexContainingKeyValueBefore(queryType, queryKey, queryValue,
		mLogStatementList.size() - 1);
		}

		/**
		* Find the index of the last LogStatement before {@code startingIndex} of type {@code type}
		* containing the given key-value pair.
		*
		* @param queryType a String that must be {@code String.equals()} to the LogStatement type
		* @param queryKey a String that must be {@code String.equals()} to a key in the LogStatement
		* @param queryValue an Object that must be {@code String.equals()} to the key's corresponding
		* value
		* @param startingIndex the index to start the backward search from. Must be less than the
		* length of mLogStatementList, or an IndexOutOfBoundsException is thrown. Can be negative,
		* in which case -1 is returned.
		*
		* @return The index of the last LogStatement, -1 if none exists.
		*/
		private int findLastIndexContainingKeyValueBefore(final String queryType, final String queryKey,
		final Object queryValue, final int startingIndex) {
		if (startingIndex < 0) {
		return -1;
		}
		for (int index = startingIndex; index >= 0; index--) {
		final LogStatement logStatement = mLogStatementList.get(index);
		final String type = logStatement.getType();
		if (type.equals(queryType) && (queryKey == null
		\|\| logStatement.containsKeyValuePair(queryKey, queryValue,
		mValuesList.get(index)))) {
		return index;
		}
		}
		return -1;
		}
		}

java/src/com/android/inputmethod/research/MotionEventReader.java

0 → 100644

+113 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2013 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License"); you may not
		* use this file except in compliance with the License. You may obtain a copy of
		* the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
		* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
		* License for the specific language governing permissions and limitations under
		* the License.
		*/

		package com.android.inputmethod.research;

		import android.util.JsonReader;
		import android.util.Log;
		import android.view.MotionEvent;

		import com.android.inputmethod.latin.define.ProductionFlag;

		import java.io.BufferedReader;
		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileNotFoundException;
		import java.io.IOException;
		import java.io.InputStreamReader;
		import java.util.ArrayList;

		public class MotionEventReader {
		private static final String TAG = MotionEventReader.class.getSimpleName();
		private static final boolean DEBUG = false && ProductionFlag.IS_EXPERIMENTAL_DEBUG;

		public ReplayData readMotionEventData(final File file) {
		final ReplayData replayData = new ReplayData();
		try {
		// Read file
		final JsonReader jsonReader = new JsonReader(new BufferedReader(new InputStreamReader(
		new FileInputStream(file))));
		jsonReader.beginArray();
		while (jsonReader.hasNext()) {
		readLogStatement(jsonReader, replayData);
		}
		jsonReader.endArray();
		} catch (FileNotFoundException e) {
		e.printStackTrace();
		} catch (IOException e) {
		e.printStackTrace();
		}
		return replayData;
		}

		static class ReplayData {
		final ArrayList<Integer> mActions = new ArrayList<Integer>();
		final ArrayList<Integer> mXCoords = new ArrayList<Integer>();
		final ArrayList<Integer> mYCoords = new ArrayList<Integer>();
		final ArrayList<Long> mTimes = new ArrayList<Long>();
		}

		private void readLogStatement(final JsonReader jsonReader, final ReplayData replayData)
		throws IOException {
		String logStatementType = null;
		Integer actionType = null;
		Integer x = null;
		Integer y = null;
		Long time = null;
		boolean loggingRelated = false;

		jsonReader.beginObject();
		while (jsonReader.hasNext()) {
		final String key = jsonReader.nextName();
		if (key.equals("_ty")) {
		logStatementType = jsonReader.nextString();
		} else if (key.equals("_ut")) {
		time = jsonReader.nextLong();
		} else if (key.equals("x")) {
		x = jsonReader.nextInt();
		} else if (key.equals("y")) {
		y = jsonReader.nextInt();
		} else if (key.equals("action")) {
		final String s = jsonReader.nextString();
		if (s.equals("UP")) {
		actionType = MotionEvent.ACTION_UP;
		} else if (s.equals("DOWN")) {
		actionType = MotionEvent.ACTION_DOWN;
		} else if (s.equals("MOVE")) {
		actionType = MotionEvent.ACTION_MOVE;
		}
		} else if (key.equals("loggingRelated")) {
		loggingRelated = jsonReader.nextBoolean();
		} else {
		if (DEBUG) {
		Log.w(TAG, "Unknown JSON key in LogStatement: " + key);
		}
		jsonReader.skipValue();
		}
		}
		jsonReader.endObject();

		if (logStatementType != null && time != null && x != null && y != null && actionType != null
		&& logStatementType.equals("MainKeyboardViewProcessMotionEvent")
		&& !loggingRelated) {
		replayData.mActions.add(actionType);
		replayData.mXCoords.add(x);
		replayData.mYCoords.add(y);
		replayData.mTimes.add(time);
		}
		}

		}

java/src/com/android/inputmethod/research/Replayer.java

0 → 100644

+120 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2012 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License"); you may not
		* use this file except in compliance with the License. You may obtain a copy of
		* the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
		* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
		* License for the specific language governing permissions and limitations under
		* the License.
		*/

		package com.android.inputmethod.research;

		import android.os.Handler;
		import android.os.Message;
		import android.os.SystemClock;
		import android.util.Log;
		import android.view.MotionEvent;

		import com.android.inputmethod.keyboard.KeyboardSwitcher;
		import com.android.inputmethod.keyboard.MainKeyboardView;
		import com.android.inputmethod.latin.define.ProductionFlag;
		import com.android.inputmethod.research.MotionEventReader.ReplayData;

		/**
		* Replays a sequence of motion events in realtime on the screen.
		*
		* Useful for user inspection of logged data.
		*/
		public class Replayer {
		private static final String TAG = Replayer.class.getSimpleName();
		private static final boolean DEBUG = false && ProductionFlag.IS_EXPERIMENTAL_DEBUG;
		private static final long START_TIME_DELAY_MS = 500;

		private boolean mIsReplaying = false;
		private KeyboardSwitcher mKeyboardSwitcher;

		public void setKeyboardSwitcher(final KeyboardSwitcher keyboardSwitcher) {
		mKeyboardSwitcher = keyboardSwitcher;
		}

		private static final int MSG_MOTION_EVENT = 0;
		private static final int MSG_DONE = 1;
		private static final int COMPLETION_TIME_MS = 500;

		// TODO: Support historical events and multi-touch.
		public void replay(final ReplayData replayData) {
		if (mIsReplaying) {
		return;
		}

		mIsReplaying = true;
		final int numActions = replayData.mActions.size();
		if (DEBUG) {
		Log.d(TAG, "replaying " + numActions + " actions");
		}
		if (numActions == 0) {
		mIsReplaying = false;
		return;
		}
		final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView();

		// The reference time relative to the times stored in events.
		final long origStartTime = replayData.mTimes.get(0);
		// The reference time relative to which events are replayed in the present.
		final long currentStartTime = SystemClock.uptimeMillis() + START_TIME_DELAY_MS;
		// The adjustment needed to translate times from the original recorded time to the current
		// time.
		final long timeAdjustment = currentStartTime - origStartTime;
		final Handler handler = new Handler() {
		// Track the time of the most recent DOWN event, to be passed as a parameter when
		// constructing a MotionEvent. It's initialized here to the origStartTime, but this is
		// only a precaution. The value should be overwritten by the first ACTION_DOWN event
		// before the first use of the variable. Note that this may cause the first few events
		// to have incorrect {@code downTime}s.
		private long mOrigDownTime = origStartTime;

		@Override
		public void handleMessage(final Message msg) {
		switch (msg.what) {
		case MSG_MOTION_EVENT:
		final int index = msg.arg1;
		final int action = replayData.mActions.get(index);
		final int x = replayData.mXCoords.get(index);
		final int y = replayData.mYCoords.get(index);
		final long origTime = replayData.mTimes.get(index);
		if (action == MotionEvent.ACTION_DOWN) {
		mOrigDownTime = origTime;
		}

		final MotionEvent me = MotionEvent.obtain(mOrigDownTime + timeAdjustment,
		origTime + timeAdjustment, action, x, y, 0);
		mainKeyboardView.processMotionEvent(me);
		me.recycle();
		break;
		case MSG_DONE:
		mIsReplaying = false;
		break;
		}
		}
		};

		for (int i = 0; i < numActions; i++) {
		final Message msg = Message.obtain(handler, MSG_MOTION_EVENT, i, 0);
		final long msgTime = replayData.mTimes.get(i) + timeAdjustment;
		handler.sendMessageAtTime(msg, msgTime);
		if (DEBUG) {
		Log.d(TAG, "queuing event at " + msgTime);
		}
		}
		final long presentDoneTime = replayData.mTimes.get(numActions - 1) + timeAdjustment
		+ COMPLETION_TIME_MS;
		handler.sendMessageAtTime(Message.obtain(handler, MSG_DONE), presentDoneTime);
		}
		}