Detect and report error when RILJ faces repetitive wakelock timeout or system errors (a0b66a3a) · Commits · e / os / android_frameworks_opt_telephony

src/java/com/android/internal/telephony/RIL.java

+31 −1

Original line number	Diff line number	Diff line
		@@ -63,6 +63,7 @@ import android.os.RemoteException;
		import android.os.SystemClock;
		import android.os.SystemProperties;
		import android.os.WorkSource;
		import android.provider.Settings;
		import android.service.carrier.CarrierIdentifier;
		import android.telephony.AccessNetworkConstants.AccessNetworkType;
		import android.telephony.CellInfo;
		@@ -206,6 +207,8 @@ public class RIL extends BaseCommands implements CommandsInterface {

		/** Telephony metrics instance for logging metrics event */
		private TelephonyMetrics mMetrics = TelephonyMetrics.getInstance();
		/** Radio bug detector instance */
		private RadioBugDetector mRadioBugDetector = null;

		boolean mIsMobileNetworkSupported;
		RadioResponse mRadioResponse;
		@@ -269,6 +272,9 @@ public class RIL extends BaseCommands implements CommandsInterface {

		synchronized (mRequestList) {
		if (msg.arg1 == mWlSequenceNum && clearWakeLock(FOR_WAKELOCK)) {
		if (mRadioBugDetector != null) {
		mRadioBugDetector.processWakelockTimeout();
		}
		if (RILJ_LOGD) {
		int count = mRequestList.size();
		Rlog.d(RILJ_LOG_TAG, "WAKE_LOCK_TIMEOUT " +
		@@ -322,6 +328,15 @@ public class RIL extends BaseCommands implements CommandsInterface {
		}
		}

		/** Return RadioBugDetector instance for testing. */
		@VisibleForTesting
		public RadioBugDetector getRadioBugDetector() {
		if (mRadioBugDetector == null) {
		mRadioBugDetector = new RadioBugDetector(mContext, mPhoneId);
		}
		return mRadioBugDetector;
		}

		/**
		* In order to prevent calls to Telephony from waiting indefinitely
		* low-latency blocking calls will eventually time out. In the event of
		@@ -535,6 +550,9 @@ public class RIL extends BaseCommands implements CommandsInterface {
		mPreferredNetworkType = preferredNetworkType;
		mPhoneType = RILConstants.NO_PHONE;
		mPhoneId = instanceId == null ? 0 : instanceId;
		if (isRadioBugDetectionEnabled()) {
		mRadioBugDetector = new RadioBugDetector(context, mPhoneId);
		}

		ConnectivityManager cm = (ConnectivityManager)context.getSystemService(
		Context.CONNECTIVITY_SERVICE);
		@@ -574,6 +592,13 @@ public class RIL extends BaseCommands implements CommandsInterface {
		}
		}

		private boolean isRadioBugDetectionEnabled() {
		return Settings.Global.getInt(
		mContext.getContentResolver(),
		Settings.Global.ENABLE_RADIO_BUG_DETECTION,
		0) != 0;
		}

		@Override
		public void setOnNITZTime(Handler h, int what, Object obj) {
		super.setOnNITZTime(h, what, obj);
		@@ -4353,6 +4378,9 @@ public class RIL extends BaseCommands implements CommandsInterface {
		Rlog.w(RILJ_LOG_TAG, "Unexpected solicited ack response! sn: " + serial);
		} else {
		decrementWakeLock(rr);
		if (mRadioBugDetector != null) {
		mRadioBugDetector.detectRadioBug(rr.mRequest, error);
		}
		if (RILJ_LOGD) {
		riljLog(rr.serialString() + " Ack < " + requestToString(rr.mRequest));
		}
		@@ -4369,7 +4397,9 @@ public class RIL extends BaseCommands implements CommandsInterface {

		// Time logging for RIL command and storing it in TelephonyHistogram.
		addToRilHistogram(rr);

		if (mRadioBugDetector != null) {
		mRadioBugDetector.detectRadioBug(rr.mRequest, error);
		}
		if (type == RadioResponseType.SOLICITED_ACK_EXP) {
		sendAck();
		if (RIL.RILJ_LOGD) {

src/java/com/android/internal/telephony/RadioBugDetector.java

0 → 100644

+170 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.internal.telephony;

		import android.content.Context;
		import android.content.Intent;
		import android.hardware.radio.V1_0.RadioError;
		import android.provider.Settings;

		import com.android.internal.annotations.VisibleForTesting;

		import java.util.HashMap;

		/**
		* This class aims to detect radio bug based on wakelock timeout and system error.
		*
		* {@hide}
		*/
		public class RadioBugDetector {
		/** Radio error constants */
		private static final int RADIO_BUG_NONE = 0x00;
		private static final int RADIO_BUG_REPETITIVE_WAKELOCK_TIMEOUT_ERROR = 0x01;
		@VisibleForTesting
		protected static final int RADIO_BUG_REPETITIVE_SYSTEM_ERROR = 0x02;

		/**
		* Default configuration values for radio bug detection.
		* The value should be large enough to avoid false alarm. From past log analysis, 10 wakelock
		* timeout took around 1 hour. 100 accumulated system_err is an estimation for abnormal radio.
		*/
		private static final int DEFAULT_WAKELOCK_TIMEOUT_COUNT_THRESHOLD = 10;
		private static final int DEFAULT_SYSTEM_ERROR_COUNT_THRESHOLD = 100;

		private Context mContext;
		private int mContinuousWakelockTimoutCount = 0;
		private int mRadioBugStatus = RADIO_BUG_NONE;
		private int mSlotId;
		private int mWakelockTimeoutThreshold = 0;
		private int mSystemErrorThreshold = 0;

		private HashMap<Integer, Integer> mSysErrRecord = new HashMap<Integer, Integer>();

		/** Constructor */
		public RadioBugDetector(Context context, int slotId) {
		mContext = context;
		mSlotId = slotId;
		init();
		}

		private void init() {
		mWakelockTimeoutThreshold = Settings.Global.getInt(
		mContext.getContentResolver(),
		Settings.Global.RADIO_BUG_WAKELOCK_TIMEOUT_COUNT_THRESHOLD,
		DEFAULT_WAKELOCK_TIMEOUT_COUNT_THRESHOLD);
		mSystemErrorThreshold = Settings.Global.getInt(
		mContext.getContentResolver(),
		Settings.Global.RADIO_BUG_SYSTEM_ERROR_COUNT_THRESHOLD,
		DEFAULT_SYSTEM_ERROR_COUNT_THRESHOLD);
		}

		/**
		* Detect radio bug and notify this issue once the threshold is reached.
		*
		* @param requestType The command type information we retrieved
		* @param error The error we received
		*/
		public synchronized void detectRadioBug(int requestType, int error) {
		/**
		* When this function is executed, it means RIL is alive. So, reset WakelockTimeoutCount.
		* Regarding SYSTEM_ERR, although RIL is alive, the connection with modem may be broken.
		* For this error, accumulate the count and check if broadcast should be sent or not.
		* For normal response or other error, reset the count of SYSTEM_ERR of the specific
		* request type and mRadioBugStatus if necessary
		*/
		mContinuousWakelockTimoutCount = 0;
		if (error == RadioError.SYSTEM_ERR) {
		int errorCount = mSysErrRecord.getOrDefault(requestType, 0);
		errorCount++;
		mSysErrRecord.put(requestType, errorCount);
		broadcastBug(true);
		} else {
		// Reset system error count if we get non-system error response.
		mSysErrRecord.remove(requestType);
		if (!isFrequentSystemError()) {
		mRadioBugStatus = RADIO_BUG_NONE;
		}
		}
		}

		/**
		* When wakelock timeout is detected, accumulate its count and check if broadcast should be
		* sent or not.
		*/
		public void processWakelockTimeout() {
		mContinuousWakelockTimoutCount++;
		broadcastBug(false);
		}

		private synchronized void broadcastBug(boolean isSystemError) {
		if (isSystemError) {
		if (!isFrequentSystemError()) {
		return;
		}
		} else {
		if (mContinuousWakelockTimoutCount < mWakelockTimeoutThreshold) {
		return;
		}
		}

		// Notify that the RIL is stuck if SYSTEM_ERR or WAKE_LOCK_TIMEOUT returned by vendor
		// RIL is more than the threshold times.
		if (mRadioBugStatus == RADIO_BUG_NONE) {
		mRadioBugStatus = isSystemError ? RADIO_BUG_REPETITIVE_SYSTEM_ERROR :
		RADIO_BUG_REPETITIVE_WAKELOCK_TIMEOUT_ERROR;
		Intent intent = new Intent(TelephonyIntents.ACTION_REPORT_RADIO_BUG);
		intent.putExtra(TelephonyIntents.EXTRA_SLOT_ID, mSlotId);
		intent.putExtra(TelephonyIntents.EXTRA_RADIO_BUG_TYPE, mRadioBugStatus);
		mContext.sendBroadcast(intent, android.Manifest.permission.READ_PRIVILEGED_PHONE_STATE);
		}
		}

		private boolean isFrequentSystemError() {
		int countForError = 0;
		boolean error = false;
		for (int count : mSysErrRecord.values()) {
		countForError += count;
		if (countForError >= mSystemErrorThreshold) {
		error = true;
		break;
		}
		}
		return error;
		}

		@VisibleForTesting
		public int getRadioBugStatus() {
		return mRadioBugStatus;
		}

		@VisibleForTesting
		public int getWakelockTimeoutThreshold() {
		return mWakelockTimeoutThreshold;
		}

		@VisibleForTesting
		public int getSystemErrorThreshold() {
		return mSystemErrorThreshold;
		}

		@VisibleForTesting
		public int getWakelockTimoutCount() {
		return mContinuousWakelockTimoutCount;
		}

		}

tests/telephonytests/src/com/android/internal/telephony/RILTest.java

+51 −0

Original line number	Diff line number	Diff line
		@@ -84,6 +84,7 @@ import static junit.framework.Assert.assertTrue;

		import static org.mockito.Matchers.any;
		import static org.mockito.Matchers.eq;
		import static org.mockito.Mockito.atLeast;
		import static org.mockito.Mockito.doReturn;
		import static org.mockito.Mockito.mock;
		import static org.mockito.Mockito.spy;
		@@ -155,6 +156,9 @@ public class RILTest extends TelephonyTest {
		// refer to RIL#DEFAULT_WAKE_LOCK_TIMEOUT_MS
		private static final int DEFAULT_WAKE_LOCK_TIMEOUT_MS = 60000;

		// timer for CountDownLatch timeout
		private static final int WAIT_TIMER = 10; //

		@Mock
		private ConnectivityManager mConnectionManager;
		@Mock
		@@ -300,6 +304,37 @@ public class RILTest extends TelephonyTest {
		super.tearDown();
		}

		@Test
		public void testRadioErrorWithWakelockTimeout() throws Exception {
		RadioBugDetector detector = mRILInstance.getRadioBugDetector();
		int wakelockTimeoutThreshold = detector.getWakelockTimeoutThreshold();
		for (int i = 0; i < wakelockTimeoutThreshold; i++) {
		invokeMethod(
		mRILInstance,
		"obtainRequest",
		new Class<?>[]{Integer.TYPE, Message.class, WorkSource.class},
		new Object[]{RIL_REQUEST_GET_SIM_STATUS, obtainMessage(), new WorkSource()});
		}

		waitForHandlerActionDelayed(mRilHandler, WAIT_TIMER, DEFAULT_WAKE_LOCK_TIMEOUT_MS);
		assertTrue(1 == detector.getWakelockTimoutCount());
		}

		@Test
		public void testRadioErrorWithContinuousSystemErr() throws Exception {
		RadioBugDetector detector = mRILUnderTest.getRadioBugDetector();
		int systemErrorThreshold = detector.getSystemErrorThreshold();
		for (int i = 0; i < systemErrorThreshold; i++) {
		mRILUnderTest.getIccCardStatus(obtainMessage());
		verify(mRadioProxy, atLeast(1)).getIccCardStatus(mSerialNumberCaptor.capture());
		verifyRILErrorResponse(mRILUnderTest, mSerialNumberCaptor.getValue(),
		RIL_REQUEST_GET_SIM_STATUS, RadioError.SYSTEM_ERR);
		}

		int status = detector.getRadioBugStatus();
		assertTrue(status == RadioBugDetector.RADIO_BUG_REPETITIVE_SYSTEM_ERROR);
		}

		@FlakyTest
		@Test
		public void testGetIccCardStatus() throws Exception {
		@@ -1081,6 +1116,22 @@ public class RILTest extends TelephonyTest {
		assertFalse(ril.getWakeLock(RIL.FOR_WAKELOCK).isHeld());
		}

		private static void verifyRILErrorResponse(RIL ril, int serial, int requestType, int error) {
		RadioResponseInfo responseInfo =
		createFakeRadioResponseInfo(serial, error, RadioResponseType.SOLICITED);

		RILRequest rr = ril.processResponse(responseInfo);
		assertNotNull(rr);

		assertEquals(serial, rr.getSerial());
		assertEquals(requestType, rr.getRequest());
		assertTrue(ril.getWakeLock(RIL.FOR_WAKELOCK).isHeld());

		ril.processResponseDone(rr, responseInfo, null);
		assertEquals(0, ril.getRilRequestList().size());
		assertFalse(ril.getWakeLock(RIL.FOR_WAKELOCK).isHeld());
		}

		private static RadioResponseInfo createFakeRadioResponseInfo(int serial, int error, int type) {
		RadioResponseInfo respInfo = new RadioResponseInfo();
		respInfo.serial = serial;