Merge changes Ieeb0ebef,I1f33a8c6,I07168a7b am: fb5d22ffea (2d273192) · Commits · e / os / android_frameworks_base

services/core/java/com/android/server/locksettings/RebootEscrowManager.java

+141 −10

Original line number	Diff line number	Diff line
		@@ -15,14 +15,17 @@
		*/

		package com.android.server.locksettings;

		import static android.os.UserHandle.USER_SYSTEM;

		import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.annotation.UserIdInt;
		import android.content.Context;
		import android.content.pm.UserInfo;
		import android.os.Handler;
		import android.os.SystemClock;
		import android.os.SystemProperties;
		import android.os.UserManager;
		import android.provider.DeviceConfig;
		import android.provider.Settings;
		@@ -35,6 +38,8 @@ import com.android.internal.util.IndentingPrintWriter;
		import com.android.internal.widget.RebootEscrowListener;

		import java.io.IOException;
		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.text.SimpleDateFormat;
		import java.util.ArrayList;
		import java.util.Date;
		@@ -65,6 +70,22 @@ class RebootEscrowManager {
		public static final String REBOOT_ESCROW_ARMED_KEY = "reboot_escrow_armed_count";

		static final String REBOOT_ESCROW_KEY_ARMED_TIMESTAMP = "reboot_escrow_key_stored_timestamp";
		static final String REBOOT_ESCROW_KEY_PROVIDER = "reboot_escrow_key_provider";

		/**
		* The verified boot 2.0 vbmeta digest of the current slot, the property value is always
		* available after boot.
		*/
		static final String VBMETA_DIGEST_PROP_NAME = "ro.boot.vbmeta.digest";
		/**
		* The system prop contains vbmeta digest of the inactive slot. The build property is set after
		* an OTA update. RebootEscrowManager will store it in disk before the OTA reboot, so the value
		* is available for vbmeta digest verification after the device reboots.
		*/
		static final String OTHER_VBMETA_DIGEST_PROP_NAME = "ota.other.vbmeta_digest";
		static final String REBOOT_ESCROW_KEY_VBMETA_DIGEST = "reboot_escrow_key_vbmeta_digest";
		static final String REBOOT_ESCROW_KEY_OTHER_VBMETA_DIGEST =
		"reboot_escrow_key_other_vbmeta_digest";

		/**
		* Number of boots until we consider the escrow data to be stale for the purposes of metrics.
		@@ -86,6 +107,31 @@ class RebootEscrowManager {
		private static final int DEFAULT_LOAD_ESCROW_DATA_RETRY_COUNT = 3;
		private static final int DEFAULT_LOAD_ESCROW_DATA_RETRY_INTERVAL_SECONDS = 30;

		@IntDef(prefix = {"ERROR_"}, value = {
		ERROR_NONE,
		ERROR_UNKNOWN,
		ERROR_NO_PROVIDER,
		ERROR_LOAD_ESCROW_KEY,
		ERROR_RETRY_COUNT_EXHAUSTED,
		ERROR_UNLOCK_ALL_USERS,
		ERROR_PROVIDER_MISMATCH,
		ERROR_KEYSTORE_FAILURE,
		})
		@Retention(RetentionPolicy.SOURCE)
		@interface RebootEscrowErrorCode {
		}

		static final int ERROR_NONE = 0;
		static final int ERROR_UNKNOWN = 1;
		static final int ERROR_NO_PROVIDER = 2;
		static final int ERROR_LOAD_ESCROW_KEY = 3;
		static final int ERROR_RETRY_COUNT_EXHAUSTED = 4;
		static final int ERROR_UNLOCK_ALL_USERS = 5;
		static final int ERROR_PROVIDER_MISMATCH = 6;
		static final int ERROR_KEYSTORE_FAILURE = 7;

		private @RebootEscrowErrorCode int mLoadEscrowDataErrorCode = ERROR_NONE;

		/**
		* Logs events for later debugging in bugreports.
		*/
		@@ -199,6 +245,10 @@ class RebootEscrowManager {
		0);
		}

		public long getCurrentTimeMillis() {
		return System.currentTimeMillis();
		}

		public int getLoadEscrowDataRetryLimit() {
		return DeviceConfig.getInt(DeviceConfig.NAMESPACE_OTA,
		"load_escrow_data_retry_count", DEFAULT_LOAD_ESCROW_DATA_RETRY_COUNT);
		@@ -221,6 +271,11 @@ class RebootEscrowManager {
		public RebootEscrowEventLog getEventLog() {
		return new RebootEscrowEventLog();
		}

		public String getVbmetaDigest(boolean other) {
		return other ? SystemProperties.get(OTHER_VBMETA_DIGEST_PROP_NAME)
		: SystemProperties.get(VBMETA_DIGEST_PROP_NAME);
		}
		}

		RebootEscrowManager(Context context, Callbacks callbacks, LockSettingsStorage storage) {
		@@ -261,6 +316,7 @@ class RebootEscrowManager {
		if (rebootEscrowUsers.isEmpty()) {
		Slog.i(TAG, "No reboot escrow data found for users,"
		+ " skipping loading escrow data");
		clearMetricsStorage();
		return;
		}

		@@ -284,6 +340,7 @@ class RebootEscrowManager {
		}

		Slog.w(TAG, "Failed to load reboot escrow data after " + attemptNumber + " attempts");
		mLoadEscrowDataErrorCode = ERROR_RETRY_COUNT_EXHAUSTED;
		onGetRebootEscrowKeyFailed(users, attemptNumber);
		}

		@@ -307,6 +364,17 @@ class RebootEscrowManager {
		}

		if (escrowKey == null) {
		if (mLoadEscrowDataErrorCode == ERROR_NONE) {
		// Specifically check if the RoR provider has changed after reboot.
		int providerType = mInjector.serverBasedResumeOnReboot()
		? RebootEscrowProviderInterface.TYPE_SERVER_BASED
		: RebootEscrowProviderInterface.TYPE_HAL;
		if (providerType != mStorage.getInt(REBOOT_ESCROW_KEY_PROVIDER, -1, USER_SYSTEM)) {
		mLoadEscrowDataErrorCode = ERROR_PROVIDER_MISMATCH;
		} else {
		mLoadEscrowDataErrorCode = ERROR_LOAD_ESCROW_KEY;
		}
		}
		onGetRebootEscrowKeyFailed(users, attemptNumber + 1);
		return;
		}
		@@ -321,9 +389,49 @@ class RebootEscrowManager {
		// Clear the old key in keystore. A new key will be generated by new RoR requests.
		mKeyStoreManager.clearKeyStoreEncryptionKey();

		if (!allUsersUnlocked && mLoadEscrowDataErrorCode == ERROR_NONE) {
		mLoadEscrowDataErrorCode = ERROR_UNLOCK_ALL_USERS;
		}
		onEscrowRestoreComplete(allUsersUnlocked, attemptNumber + 1);
		}

		private void clearMetricsStorage() {
		mStorage.removeKey(REBOOT_ESCROW_ARMED_KEY, USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_KEY_VBMETA_DIGEST, USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_KEY_OTHER_VBMETA_DIGEST, USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_KEY_PROVIDER, USER_SYSTEM);
		}

		private int getVbmetaDigestStatusOnRestoreComplete() {
		String currentVbmetaDigest = mInjector.getVbmetaDigest(false);
		String vbmetaDigestStored = mStorage.getString(REBOOT_ESCROW_KEY_VBMETA_DIGEST,
		"", USER_SYSTEM);
		String vbmetaDigestOtherStored = mStorage.getString(REBOOT_ESCROW_KEY_OTHER_VBMETA_DIGEST,
		"", USER_SYSTEM);

		// The other vbmeta digest is never set, assume no slot switch is attempted.
		if (vbmetaDigestOtherStored.isEmpty()) {
		if (currentVbmetaDigest.equals(vbmetaDigestStored)) {
		return FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MATCH_EXPECTED_SLOT;
		}
		return FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MISMATCH;
		}

		// The other vbmeta digest is set, we expect to boot into the new slot.
		if (currentVbmetaDigest.equals(vbmetaDigestOtherStored)) {
		return FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MATCH_EXPECTED_SLOT;
		} else if (currentVbmetaDigest.equals(vbmetaDigestStored)) {
		return FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MATCH_FALLBACK_SLOT;
		}
		return FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MISMATCH;
		}

		private void reportMetricOnRestoreComplete(boolean success, int attemptCount) {
		int serviceType = mInjector.serverBasedResumeOnReboot()
		? FrameworkStatsLog.REBOOT_ESCROW_RECOVERY_REPORTED__TYPE__SERVER_BASED
		@@ -331,26 +439,32 @@ class RebootEscrowManager {

		long armedTimestamp = mStorage.getLong(REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, -1,
		USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, USER_SYSTEM);
		int escrowDurationInSeconds = armedTimestamp != -1
		? (int) (System.currentTimeMillis() - armedTimestamp) / 1000 : -1;
		int escrowDurationInSeconds = -1;
		long currentTimeStamp = mInjector.getCurrentTimeMillis();
		if (armedTimestamp != -1 && currentTimeStamp > armedTimestamp) {
		escrowDurationInSeconds = (int) (currentTimeStamp - armedTimestamp) / 1000;
		}

		// TODO(b/179105110) design error code; and report the true value for other fields.
		int vbmetaDigestStatus = FrameworkStatsLog
		.REBOOT_ESCROW_RECOVERY_REPORTED__VBMETA_DIGEST_STATUS__MATCH_EXPECTED_SLOT;
		int vbmetaDigestStatus = getVbmetaDigestStatusOnRestoreComplete();
		if (!success && mLoadEscrowDataErrorCode == ERROR_NONE) {
		mLoadEscrowDataErrorCode = ERROR_UNKNOWN;
		}

		mInjector.reportMetric(success, 0 /* error code */, serviceType, attemptCount,
		// TODO(179105110) report the duration since boot complete.
		mInjector.reportMetric(success, mLoadEscrowDataErrorCode, serviceType, attemptCount,
		escrowDurationInSeconds, vbmetaDigestStatus, -1);

		mLoadEscrowDataErrorCode = ERROR_NONE;
		}

		private void onEscrowRestoreComplete(boolean success, int attemptCount) {
		int previousBootCount = mStorage.getInt(REBOOT_ESCROW_ARMED_KEY, -1, USER_SYSTEM);
		mStorage.removeKey(REBOOT_ESCROW_ARMED_KEY, USER_SYSTEM);

		int bootCountDelta = mInjector.getBootCount() - previousBootCount;
		if (success \|\| (previousBootCount != -1 && bootCountDelta <= BOOT_COUNT_TOLERANCE)) {
		reportMetricOnRestoreComplete(success, attemptCount);
		}
		clearMetricsStorage();
		}

		private RebootEscrowKey getAndClearRebootEscrowKey(SecretKey kk) throws IOException {
		@@ -358,6 +472,14 @@ class RebootEscrowManager {
		if (rebootEscrowProvider == null) {
		Slog.w(TAG,
		"Had reboot escrow data for users, but RebootEscrowProvider is unavailable");
		mLoadEscrowDataErrorCode = ERROR_NO_PROVIDER;
		return null;
		}

		// Server based RoR always need the decryption key from keystore.
		if (rebootEscrowProvider.getType() == RebootEscrowProviderInterface.TYPE_SERVER_BASED
		&& kk == null) {
		mLoadEscrowDataErrorCode = ERROR_KEYSTORE_FAILURE;
		return null;
		}

		@@ -463,7 +585,7 @@ class RebootEscrowManager {
		return;
		}

		mStorage.removeKey(REBOOT_ESCROW_ARMED_KEY, USER_SYSTEM);
		clearMetricsStorage();
		rebootEscrowProvider.clearRebootEscrowKey();

		List<UserInfo> users = mUserManager.getUsers();
		@@ -486,6 +608,9 @@ class RebootEscrowManager {
		return false;
		}

		int actualProviderType = rebootEscrowProvider.getType();
		// TODO(b/183140900) Fail the reboot if provider type mismatches.

		RebootEscrowKey escrowKey;
		synchronized (mKeyGenerationLock) {
		escrowKey = mPendingRebootEscrowKey;
		@@ -505,8 +630,14 @@ class RebootEscrowManager {
		boolean armedRebootEscrow = rebootEscrowProvider.storeRebootEscrowKey(escrowKey, kk);
		if (armedRebootEscrow) {
		mStorage.setInt(REBOOT_ESCROW_ARMED_KEY, mInjector.getBootCount(), USER_SYSTEM);
		mStorage.setLong(REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, System.currentTimeMillis(),
		mStorage.setLong(REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, mInjector.getCurrentTimeMillis(),
		USER_SYSTEM);
		// Store the vbmeta digest of both slots.
		mStorage.setString(REBOOT_ESCROW_KEY_VBMETA_DIGEST, mInjector.getVbmetaDigest(false),
		USER_SYSTEM);
		mStorage.setString(REBOOT_ESCROW_KEY_OTHER_VBMETA_DIGEST,
		mInjector.getVbmetaDigest(true), USER_SYSTEM);
		mStorage.setInt(REBOOT_ESCROW_KEY_PROVIDER, actualProviderType, USER_SYSTEM);
		mEventLog.addEntry(RebootEscrowEvent.SET_ARMED_STATUS);
		}

services/core/java/com/android/server/locksettings/RebootEscrowProviderHalImpl.java

+5 −0

Original line number	Diff line number	Diff line
		@@ -59,6 +59,11 @@ class RebootEscrowProviderHalImpl implements RebootEscrowProviderInterface {
		mInjector = injector;
		}

		@Override
		public int getType() {
		return TYPE_HAL;
		}

		@Override
		public boolean hasRebootEscrowSupport() {
		return mInjector.getRebootEscrow() != null;

services/core/java/com/android/server/locksettings/RebootEscrowProviderInterface.java

+19 −0

Original line number	Diff line number	Diff line
		@@ -16,7 +16,11 @@

		package com.android.server.locksettings;

		import android.annotation.IntDef;

		import java.io.IOException;
		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;

		import javax.crypto.SecretKey;

		@@ -28,6 +32,21 @@ import javax.crypto.SecretKey;
		* @hide
		*/
		public interface RebootEscrowProviderInterface {
		@IntDef(prefix = {"TYPE_"}, value = {
		TYPE_HAL,
		TYPE_SERVER_BASED,
		})
		@Retention(RetentionPolicy.SOURCE)
		@interface RebootEscrowProviderType {
		}
		int TYPE_HAL = 0;
		int TYPE_SERVER_BASED = 1;

		/**
		* Returns the reboot escrow provider type.
		*/
		@RebootEscrowProviderType int getType();

		/**
		* Returns true if the secure store/discard of reboot escrow key is supported.
		*/

services/core/java/com/android/server/locksettings/RebootEscrowProviderServerBasedImpl.java

+5 −0

Original line number	Diff line number	Diff line
		@@ -94,6 +94,11 @@ class RebootEscrowProviderServerBasedImpl implements RebootEscrowProviderInterfa
		mInjector = injector;
		}

		@Override
		public int getType() {
		return TYPE_SERVER_BASED;
		}

		@Override
		public boolean hasRebootEscrowSupport() {
		return mInjector.getServiceConnection() != null;

services/tests/servicestests/src/com/android/server/locksettings/RebootEscrowManagerTests.java

+118 −6

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ import static android.content.pm.UserInfo.FLAG_PRIMARY;
		import static android.content.pm.UserInfo.FLAG_PROFILE;
		import static android.os.UserHandle.USER_SYSTEM;

		import static org.junit.Assert.assertEquals;
		import static org.junit.Assert.assertFalse;
		import static org.junit.Assert.assertNotNull;
		import static org.junit.Assert.assertTrue;
		@@ -110,6 +111,10 @@ public class RebootEscrowManagerTests {
		public interface MockableRebootEscrowInjected {
		int getBootCount();

		long getCurrentTimeMillis();

		boolean forceServerBased();

		void reportMetric(boolean success, int errorCode, int serviceType, int attemptCount,
		int escrowDurationInSeconds, int vbmetaDigestStatus, int durationSinceBootComplete);
		}
		@@ -174,6 +179,9 @@ public class RebootEscrowManagerTests {

		@Override
		public boolean serverBasedResumeOnReboot() {
		if (mInjected.forceServerBased()) {
		return true;
		}
		return mServerBased;
		}

		@@ -204,10 +212,21 @@ public class RebootEscrowManagerTests {
		return 1;
		}

		@Override
		public String getVbmetaDigest(boolean other) {
		return other ? "" : "fake digest";
		}

		@Override
		public long getCurrentTimeMillis() {
		return mInjected.getCurrentTimeMillis();
		}

		@Override
		public void reportMetric(boolean success, int errorCode, int serviceType, int attemptCount,
		int escrowDurationInSeconds, int vbmetaDigestStatus,
		int durationSinceBootComplete) {

		mInjected.reportMetric(success, errorCode, serviceType, attemptCount,
		escrowDurationInSeconds, vbmetaDigestStatus, durationSinceBootComplete);
		}
		@@ -430,16 +449,21 @@ public class RebootEscrowManagerTests {
		// pretend reboot happens here

		when(mInjected.getBootCount()).thenReturn(1);
		when(mInjected.getCurrentTimeMillis()).thenReturn(30000L);
		mStorage.setLong(RebootEscrowManager.REBOOT_ESCROW_KEY_ARMED_TIMESTAMP, 10000L,
		USER_SYSTEM);
		ArgumentCaptor<Boolean> metricsSuccessCaptor = ArgumentCaptor.forClass(Boolean.class);
		doNothing().when(mInjected).reportMetric(metricsSuccessCaptor.capture(),
		eq(0) /* error code /, eq(1) / HAL based /, eq(1) / attempt count */,
		anyInt(), anyInt(), anyInt());
		eq(20), eq(0) /* vbmeta status */, anyInt());
		when(mRebootEscrow.retrieveKey()).thenAnswer(invocation -> keyByteCaptor.getValue());

		mService.loadRebootEscrowDataIfAvailable(null);
		verify(mRebootEscrow).retrieveKey();
		assertTrue(metricsSuccessCaptor.getValue());
		verify(mKeyStoreManager).clearKeyStoreEncryptionKey();
		assertEquals(mStorage.getLong(RebootEscrowManager.REBOOT_ESCROW_KEY_ARMED_TIMESTAMP,
		-1, USER_SYSTEM), -1);
		}

		@Test
		@@ -468,7 +492,7 @@ public class RebootEscrowManagerTests {
		ArgumentCaptor<Boolean> metricsSuccessCaptor = ArgumentCaptor.forClass(Boolean.class);
		doNothing().when(mInjected).reportMetric(metricsSuccessCaptor.capture(),
		eq(0) /* error code /, eq(2) / Server based /, eq(1) / attempt count */,
		anyInt(), anyInt(), anyInt());
		anyInt(), eq(0) /* vbmeta status */, anyInt());

		when(mServiceConnection.unwrap(any(), anyLong()))
		.thenAnswer(invocation -> invocation.getArgument(0));
		@@ -478,6 +502,84 @@ public class RebootEscrowManagerTests {
		verify(mKeyStoreManager).clearKeyStoreEncryptionKey();
		}

		@Test
		public void loadRebootEscrowDataIfAvailable_ServerBasedRemoteException_Failure()
		throws Exception {
		setServerBasedRebootEscrowProvider();

		when(mInjected.getBootCount()).thenReturn(0);
		RebootEscrowListener mockListener = mock(RebootEscrowListener.class);
		mService.setRebootEscrowListener(mockListener);
		mService.prepareRebootEscrow();

		clearInvocations(mServiceConnection);
		mService.callToRebootEscrowIfNeeded(PRIMARY_USER_ID, FAKE_SP_VERSION, FAKE_AUTH_TOKEN);
		verify(mockListener).onPreparedForReboot(eq(true));
		verify(mServiceConnection, never()).wrapBlob(any(), anyLong(), anyLong());

		// Use x -> x for both wrap & unwrap functions.
		when(mServiceConnection.wrapBlob(any(), anyLong(), anyLong()))
		.thenAnswer(invocation -> invocation.getArgument(0));
		assertTrue(mService.armRebootEscrowIfNeeded());
		verify(mServiceConnection).wrapBlob(any(), anyLong(), anyLong());
		assertTrue(mStorage.hasRebootEscrowServerBlob());

		// pretend reboot happens here
		when(mInjected.getBootCount()).thenReturn(1);
		ArgumentCaptor<Boolean> metricsSuccessCaptor = ArgumentCaptor.forClass(Boolean.class);
		ArgumentCaptor<Integer> metricsErrorCodeCaptor = ArgumentCaptor.forClass(Integer.class);
		doNothing().when(mInjected).reportMetric(metricsSuccessCaptor.capture(),
		metricsErrorCodeCaptor.capture(), eq(2) /* Server based */,
		eq(1) /* attempt count /, anyInt(), eq(0) / vbmeta status */, anyInt());

		when(mServiceConnection.unwrap(any(), anyLong())).thenThrow(RemoteException.class);
		mService.loadRebootEscrowDataIfAvailable(null);
		verify(mServiceConnection).unwrap(any(), anyLong());
		assertFalse(metricsSuccessCaptor.getValue());
		assertEquals(Integer.valueOf(RebootEscrowManager.ERROR_LOAD_ESCROW_KEY),
		metricsErrorCodeCaptor.getValue());
		}

		@Test
		public void loadRebootEscrowDataIfAvailable_ServerBasedIoError_RetryFailure() throws Exception {
		setServerBasedRebootEscrowProvider();

		when(mInjected.getBootCount()).thenReturn(0);
		RebootEscrowListener mockListener = mock(RebootEscrowListener.class);
		mService.setRebootEscrowListener(mockListener);
		mService.prepareRebootEscrow();

		clearInvocations(mServiceConnection);
		mService.callToRebootEscrowIfNeeded(PRIMARY_USER_ID, FAKE_SP_VERSION, FAKE_AUTH_TOKEN);
		verify(mockListener).onPreparedForReboot(eq(true));
		verify(mServiceConnection, never()).wrapBlob(any(), anyLong(), anyLong());

		// Use x -> x for both wrap & unwrap functions.
		when(mServiceConnection.wrapBlob(any(), anyLong(), anyLong()))
		.thenAnswer(invocation -> invocation.getArgument(0));
		assertTrue(mService.armRebootEscrowIfNeeded());
		verify(mServiceConnection).wrapBlob(any(), anyLong(), anyLong());
		assertTrue(mStorage.hasRebootEscrowServerBlob());

		// pretend reboot happens here
		when(mInjected.getBootCount()).thenReturn(1);
		ArgumentCaptor<Boolean> metricsSuccessCaptor = ArgumentCaptor.forClass(Boolean.class);
		ArgumentCaptor<Integer> metricsErrorCodeCaptor = ArgumentCaptor.forClass(Integer.class);
		doNothing().when(mInjected).reportMetric(metricsSuccessCaptor.capture(),
		metricsErrorCodeCaptor.capture(), eq(2) /* Server based */,
		eq(2) /* attempt count /, anyInt(), eq(0) / vbmeta status */, anyInt());
		when(mServiceConnection.unwrap(any(), anyLong())).thenThrow(IOException.class);

		HandlerThread thread = new HandlerThread("RebootEscrowManagerTest");
		thread.start();
		mService.loadRebootEscrowDataIfAvailable(new Handler(thread.getLooper()));
		// Sleep 5s for the retry to complete
		Thread.sleep(5 * 1000);
		assertFalse(metricsSuccessCaptor.getValue());
		assertEquals(Integer.valueOf(RebootEscrowManager.ERROR_RETRY_COUNT_EXHAUSTED),
		metricsErrorCodeCaptor.getValue());
		}

		@Test
		public void loadRebootEscrowDataIfAvailable_ServerBased_RetrySuccess() throws Exception {
		setServerBasedRebootEscrowProvider();
		@@ -607,9 +709,14 @@ public class RebootEscrowManagerTests {
		when(mInjected.getBootCount()).thenReturn(10);
		when(mRebootEscrow.retrieveKey()).thenAnswer(invocation -> keyByteCaptor.getValue());

		// Trigger a vbmeta digest mismatch
		mStorage.setString(RebootEscrowManager.REBOOT_ESCROW_KEY_VBMETA_DIGEST,
		"non sense value", USER_SYSTEM);
		mService.loadRebootEscrowDataIfAvailable(null);
		verify(mInjected).reportMetric(eq(true), eq(0) /* error code /, eq(1) / HAL based */,
		eq(1) /* attempt count */, anyInt(), anyInt(), anyInt());
		eq(1) /* attempt count /, anyInt(), eq(2) / vbmeta status */, anyInt());
		assertEquals(mStorage.getString(RebootEscrowManager.REBOOT_ESCROW_KEY_VBMETA_DIGEST,
		"", USER_SYSTEM), "");
		}

		@Test
		@@ -636,12 +743,17 @@ public class RebootEscrowManagerTests {

		when(mInjected.getBootCount()).thenReturn(1);
		ArgumentCaptor<Boolean> metricsSuccessCaptor = ArgumentCaptor.forClass(Boolean.class);
		ArgumentCaptor<Integer> metricsErrorCodeCaptor = ArgumentCaptor.forClass(Integer.class);
		// Return a null escrow key
		doNothing().when(mInjected).reportMetric(metricsSuccessCaptor.capture(),
		anyInt() /* error code /, eq(1) / HAL based /, eq(1) / attempt count */,
		anyInt(), anyInt(), anyInt());
		when(mRebootEscrow.retrieveKey()).thenAnswer(invocation -> new byte[32]);
		metricsErrorCodeCaptor.capture(), eq(1) /* HAL based */,
		eq(1) /* attempt count */, anyInt(), anyInt(), anyInt());

		when(mRebootEscrow.retrieveKey()).thenAnswer(invocation -> null);
		mService.loadRebootEscrowDataIfAvailable(null);
		verify(mRebootEscrow).retrieveKey();
		assertFalse(metricsSuccessCaptor.getValue());
		assertEquals(Integer.valueOf(RebootEscrowManager.ERROR_LOAD_ESCROW_KEY),
		metricsErrorCodeCaptor.getValue());
		}
		}