Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 801edc9f authored by Eric Biggers's avatar Eric Biggers
Browse files

Rename wrong_guess_counter to failure_counter

I named the file "wrong_guess_counter" with the intention that it count
only wrong guesses, not other failures.  That would be ideal; however,
unfortunately the error codes from Weaver and Gatekeeper do not always
differentiate wrong guesses from other failures.  Therefore, the counter
does count other failures, because not rate-limiting them would make the
software rate-limiter ineffective on many devices.

Therefore, rename the file to "failure_counter".  Update code and
comments accordingly.  No functional change except the name of the file.

Note that since "failure_counter" is a less specific name anyway, this
still leaves the door open to changing exactly what types of failures
are counted in the future, including making it precisely count wrong
guesses as was my original intent.

Since this change touches the SyntheticPasswordManager class comment,
also reformat that comment to be compatible with google-java-format.

Test: atest FrameworksServicesTests:com.android.server.locksettings
Bug: 395976735
Flag: android.security.software_ratelimiter
Change-Id: I3ec9d412b70e32530acfa1352cedefed6ff29f25
parent 1f34ce76
Loading
Loading
Loading
Loading
+4 −4
Original line number Diff line number Diff line
@@ -661,13 +661,13 @@ public class LockSettingsService extends ILockSettings.Stub {
    private class SoftwareRateLimiterInjector implements SoftwareRateLimiter.Injector {

        @Override
        public int readWrongGuessCounter(LskfIdentifier id) {
            return mSpManager.readWrongGuessCounter(id);
        public int readFailureCounter(LskfIdentifier id) {
            return mSpManager.readFailureCounter(id);
        }

        @Override
        public void writeWrongGuessCounter(LskfIdentifier id, int count) {
            mSpManager.writeWrongGuessCounter(id, count);
        public void writeFailureCounter(LskfIdentifier id, int count) {
            mSpManager.writeFailureCounter(id, count);
        }

        @Override
+44 −48
Original line number Diff line number Diff line
@@ -84,8 +84,8 @@ class SoftwareRateLimiter {
    @VisibleForTesting static final Duration SAVED_WRONG_GUESS_TIMEOUT = Duration.ofMinutes(5);

    /**
     * A table that maps the number of (real) wrong guesses to the delay that is enforced after that
     * number of (real) wrong guesses. Out-of-bounds indices default to the final delay.
     * A table that maps the number of (real) failures to the delay that is enforced after that
     * number of (real) failures. Out-of-bounds indices default to the final delay.
     */
    private static final Duration[] DELAY_TABLE =
            new Duration[] {
@@ -127,14 +127,11 @@ class SoftwareRateLimiter {
    private static class RateLimiterState {

        /**
         * The number of (real) wrong guesses since the last correct guess. This includes only wrong
         * guesses that reached the hardware rate-limiter and were reported to be wrong guesses. It
         * excludes guesses that never reached the real credential check for a reason such as
         * detection of a duplicate wrong guess, too short, delay still remaining, etc. It also
         * excludes any times that the real credential check failed due to a transient error (e.g.
         * failure to communicate with the Secure Element) rather than due to the guess being wrong.
         * The number of failed attempts since the last correct guess, not counting attempts that
         * never reached the real credential check for a reason such as detection of a duplicate
         * wrong guess, too short, delay still remaining, etc.
         */
        public int numWrongGuesses;
        public int numFailures;

        /** The number of duplicate wrong guesses since the last correct guess or reboot */
        public int numDuplicateWrongGuesses;
@@ -143,10 +140,10 @@ class SoftwareRateLimiter {
        public final int statsCredentialType;

        /**
         * The time since boot at which the number of wrong guesses was last incremented, or zero if
         * the number of wrong guesses was last incremented before the current boot.
         * The time since boot at which the failure counter was last incremented, or zero if the
         * failure counter was last incremented before the current boot.
         */
        public Duration timeSinceBootOfLastWrongGuess = Duration.ZERO;
        public Duration timeSinceBootOfLastFailure = Duration.ZERO;

        /**
         * The list of wrong guesses that were recently tried already in the current boot, ordered
@@ -155,8 +152,8 @@ class SoftwareRateLimiter {
        public final LockscreenCredential[] savedWrongGuesses =
                new LockscreenCredential[MAX_SAVED_WRONG_GUESSES];

        RateLimiterState(int numWrongGuesses, LockscreenCredential firstGuess) {
            this.numWrongGuesses = numWrongGuesses;
        RateLimiterState(int numFailures, LockscreenCredential firstGuess) {
            this.numFailures = numFailures;
            this.statsCredentialType = getStatsCredentialType(firstGuess);
        }
    }
@@ -173,8 +170,8 @@ class SoftwareRateLimiter {
    private Duration getCurrentDelay(RateLimiterState state) {
        if (!mEnforcing) {
            return Duration.ZERO;
        } else if (state.numWrongGuesses >= 0 && state.numWrongGuesses < DELAY_TABLE.length) {
            return DELAY_TABLE[state.numWrongGuesses];
        } else if (state.numFailures >= 0 && state.numFailures < DELAY_TABLE.length) {
            return DELAY_TABLE[state.numFailures];
        } else {
            return DELAY_TABLE[DELAY_TABLE.length - 1];
        }
@@ -211,16 +208,15 @@ class SoftwareRateLimiter {
                            // The state isn't cached yet. Create it.
                            //
                            // For LSKF-based synthetic password protectors the only persistent
                            // software rate-limiter state is the number of wrong guesses, which is
                            // loaded from a counter file on-disk. timeSinceBootOfLastWrongGuess is
                            // just set to zero, so effectively the delay resets to its original
                            // value (for the current number of wrong guesses) upon reboot. That
                            // matches what typical hardware rate-limiter implementations do; they
                            // typically do not have access to a trusted real-time clock that runs
                            // without the device being powered on.
                            // software rate-limiter state is the failure counter.
                            // timeSinceBootOfLastFailure is just set to zero, so effectively the
                            // delay resets to its original value (for the current failure count)
                            // upon reboot. That matches what typical hardware rate-limiter
                            // implementations do; they typically do not have access to a trusted
                            // real-time clock that runs without the device being powered on.
                            //
                            // Likewise, rebooting causes any saved wrong guesses to be forgotten.
                            return new RateLimiterState(readWrongGuessCounter(id), guess);
                            return new RateLimiterState(readFailureCounter(id), guess);
                        });

        // Check for remaining delay. Note that the case of a positive remaining delay normally
@@ -230,12 +226,12 @@ class SoftwareRateLimiter {
        // following a reboot which causes the lock screen to "forget" the delay.
        final Duration delay = getCurrentDelay(state);
        final Duration now = mInjector.getTimeSinceBoot();
        final Duration remainingDelay = state.timeSinceBootOfLastWrongGuess.plus(delay).minus(now);
        final Duration remainingDelay = state.timeSinceBootOfLastFailure.plus(delay).minus(now);
        if (remainingDelay.isPositive()) {
            Slogf.e(
                    TAG,
                    "Rate-limited; numWrongGuesses=%d, remainingDelay=%s",
                    state.numWrongGuesses,
                    "Rate-limited; numFailures=%d, remainingDelay=%s",
                    state.numFailures,
                    remainingDelay);
            return SoftwareRateLimiterResult.rateLimited(remainingDelay);
        }
@@ -264,19 +260,19 @@ class SoftwareRateLimiter {
    }

    /**
     * Reports a successful guess to the software rate-limiter. This causes the wrong guess counter
     * and saved wrong guesses to be cleared.
     * Reports a successful guess to the software rate-limiter. This causes the failure counter and
     * saved wrong guesses to be cleared.
     */
    synchronized void reportSuccess(LskfIdentifier id) {
        RateLimiterState state = getExistingState(id);
        writeStats(id, state, /* success= */ true);
        // If the wrong guess counter is still 0, then there is no need to write it. Nor can there
        // be any saved wrong guesses, so there is no need to forget them. This optimizes for the
        // If the failure counter is still 0, then there is no need to write it. Nor can there be
        // any saved wrong guesses, so there is no need to forget them. This optimizes for the
        // common case where the first guess is correct.
        if (state.numWrongGuesses != 0) {
            state.numWrongGuesses = 0;
        if (state.numFailures != 0) {
            state.numFailures = 0;
            state.numDuplicateWrongGuesses = 0;
            writeWrongGuessCounter(id, state);
            writeFailureCounter(id, state);
            forgetSavedWrongGuesses(state);
        }
    }
@@ -329,13 +325,13 @@ class SoftwareRateLimiter {
        // ineffective on all such devices, still apply it. This does mean that correct guesses that
        // encountered an error will be rate-limited. However, by design the rate-limiter kicks in
        // gradually anyway, so there will be a chance for the user to try again.
        state.numWrongGuesses++;
        state.timeSinceBootOfLastWrongGuess = mInjector.getTimeSinceBoot();
        state.numFailures++;
        state.timeSinceBootOfLastFailure = mInjector.getTimeSinceBoot();

        // Update the counter on-disk. It is important that this be done before the failure is
        // reported to the UI, and that it be done synchronously e.g. by fsync()-ing the file and
        // its containing directory. This minimizes the risk of the counter being rolled back.
        writeWrongGuessCounter(id, state);
        writeFailureCounter(id, state);

        writeStats(id, state, /* success= */ false);

@@ -386,7 +382,7 @@ class SoftwareRateLimiter {
        FrameworkStatsLog.write(
                FrameworkStatsLog.LSKF_AUTHENTICATION_ATTEMPTED,
                /* success= */ success,
                /* num_unique_guesses= */ state.numWrongGuesses + (success ? 1 : 0),
                /* num_unique_guesses= */ state.numFailures + (success ? 1 : 0),
                /* num_duplicate_guesses= */ state.numDuplicateWrongGuesses,
                /* credential_type= */ state.statsCredentialType,
                /* software_rate_limiter_enforcing= */ mEnforcing,
@@ -450,22 +446,22 @@ class SoftwareRateLimiter {
        }
    }

    private int readWrongGuessCounter(LskfIdentifier id) {
    private int readFailureCounter(LskfIdentifier id) {
        if (id.isSpecialCredential()) {
            // Special credentials (e.g. FRP credential and repair mode exit credential) do not yet
            // store a persistent wrong guess counter.
            // store a persistent failure counter.
            return 0;
        }
        return mInjector.readWrongGuessCounter(id);
        return mInjector.readFailureCounter(id);
    }

    private void writeWrongGuessCounter(LskfIdentifier id, RateLimiterState state) {
    private void writeFailureCounter(LskfIdentifier id, RateLimiterState state) {
        if (id.isSpecialCredential()) {
            // Special credentials (e.g. FRP credential and repair mode exit credential) do not yet
            // store a persistent wrong guess counter.
            // store a persistent failure counter.
            return;
        }
        mInjector.writeWrongGuessCounter(id, state.numWrongGuesses);
        mInjector.writeFailureCounter(id, state.numFailures);
    }

    // Only for unit tests.
@@ -483,10 +479,10 @@ class SoftwareRateLimiter {
                            "userId=%d, protectorId=%016x", lskfId.userId, lskfId.protectorId));
            final RateLimiterState state = mState.valueAt(index);
            pw.increaseIndent();
            pw.println("numWrongGuesses=" + state.numWrongGuesses);
            pw.println("numFailures=" + state.numFailures);
            pw.println("numDuplicateWrongGuesses=" + state.numDuplicateWrongGuesses);
            pw.println("statsCredentialType=" + state.statsCredentialType);
            pw.println("timeSinceBootOfLastWrongGuess=" + state.timeSinceBootOfLastWrongGuess);
            pw.println("timeSinceBootOfLastFailure=" + state.timeSinceBootOfLastFailure);
            pw.println(
                    "numSavedWrongGuesses="
                            + Arrays.stream(state.savedWrongGuesses)
@@ -497,9 +493,9 @@ class SoftwareRateLimiter {
    }

    interface Injector {
        int readWrongGuessCounter(LskfIdentifier id);
        int readFailureCounter(LskfIdentifier id);

        void writeWrongGuessCounter(LskfIdentifier id, int count);
        void writeFailureCounter(LskfIdentifier id, int count);

        Duration getTimeSinceBoot();

+61 −55
Original line number Diff line number Diff line
@@ -81,53 +81,59 @@ import java.util.Set;
 * A class that manages a user's synthetic password (SP) ({@link #SyntheticPassword}), along with a
 * set of SP protectors that are independent ways that the SP is protected.
 *
 * Invariants for SPs:
 * <p>Invariants for SPs:
 *
 *  - A user's SP never changes, but SP protectors can be added and removed.  There is always a
 * <ul>
 *   <li>A user's SP never changes, but SP protectors can be added and removed. There is always a
 *       protector that protects the SP with the user's Lock Screen Knowledge Factor (LSKF), a.k.a.
 *       LockscreenCredential. The LSKF may be empty (none). There may be escrow token-based
 *       protectors as well, only for specific use cases such as enterprise-managed users.
 *
 *  - The user's credential-encrypted storage is always protected by the SP.
 *
 *  - The user's Keystore superencryption keys are always protected by the SP.  These in turn
 *   <li>The user's credential-encrypted storage is always protected by the SP.
 *   <li>The user's Keystore superencryption keys are always protected by the SP. These in turn
 *       protect the Keystore keys that require user authentication, an unlocked device, or both.
 *   <li>A secret derived from the synthetic password is enrolled in Gatekeeper for the user, but
 *       only while the user has a (nonempty) LSKF. This enrollment has an associated ID called the
 *       Secure user ID or SID. This use of Gatekeeper, which is separate from the use of GateKeeper
 *       that may be used in the LSKF-based protector, makes it so that unlocking the synthetic
 *       password generates a HardwareAuthToken (but only when the user has LSKF). That
 *       HardwareAuthToken can be provided to KeyMint to authorize the use of the user's
 *       authentication-bound Keystore keys.
 * </ul>
 *
 *  - A secret derived from the synthetic password is enrolled in Gatekeeper for the user, but only
 *    while the user has a (nonempty) LSKF.  This enrollment has an associated ID called the Secure
 *    user ID or SID.  This use of Gatekeeper, which is separate from the use of GateKeeper that may
 *    be used in the LSKF-based protector, makes it so that unlocking the synthetic password
 *    generates a HardwareAuthToken (but only when the user has LSKF).  That HardwareAuthToken can
 *    be provided to KeyMint to authorize the use of the user's authentication-bound Keystore keys.
 * <p>Files stored on disk for each user:
 *
 * Files stored on disk for each user:
 *   For the SP itself, stored under NULL_PROTECTOR_ID:
 *     SP_HANDLE_NAME: GateKeeper password handle of a password derived from the SP.  Only exists
 *                     while the LSKF is nonempty.
 *     SP_E0_NAME, SP_P1_NAME: Information needed to create and use escrow token-based protectors.
 *                             Deleted when escrow token support is disabled for the user.
 *     VENDOR_AUTH_SECRET_NAME: A copy of the secret passed using the IAuthSecret interface,
 * <ul>
 *   <li>For the SP itself, stored under NULL_PROTECTOR_ID:
 *       <ul>
 *         <li>SP_HANDLE_NAME: GateKeeper password handle of a password derived from the SP. Only
 *             exists while the LSKF is nonempty.
 *         <li>SP_E0_NAME, SP_P1_NAME: Information needed to create and use escrow token-based
 *             protectors. Deleted when escrow token support is disabled for the user.
 *         <li>VENDOR_AUTH_SECRET_NAME: A copy of the secret passed using the IAuthSecret interface,
 *             encrypted using a secret derived from the SP using
 *             PERSONALIZATION_AUTHSECRET_ENCRYPTION_KEY.
 *
 *     For each protector, stored under the corresponding protector ID:
 *       SP_BLOB_NAME: The encrypted SP secret (the SP itself or the P0 value).  Always exists.
 *       PASSWORD_DATA_NAME: Data used for LSKF verification, such as the scrypt salt and
 *                           parameters.  Only exists for LSKF-based protectors.  Doesn't exist when
 *                           the LSKF is empty, except in old protectors.
 *       PASSWORD_METRICS_NAME: Metrics about the LSKF, encrypted by a key derived from the SP.
 *                              Only exists for LSKF-based protectors.  Doesn't exist when the LSKF
 *                              is empty, except in old protectors.
 *       SECDISCARDABLE_NAME: A large number of random bytes that all need to be known in order to
 *                            decrypt SP_BLOB_NAME.  When the protector is deleted, this file is
 *                            overwritten and deleted as a "best-effort" attempt to support secure
 *                            deletion when hardware support for secure deletion is unavailable.
 *                            Doesn't exist for LSKF-based protectors that use Weaver.
 *       WEAVER_SLOT_NAME: Contains the Weaver slot number used by this protector.  Only exists if
 *                         the protector uses Weaver.
 *       WRONG_GUESS_COUNTER_NAME: Contains the wrong guess counter for the software rate-limiter.
 *                                 Only exists for LSKF-based protectors.  Does not affect the
 *                                 hardware rate-limiter which operates concurrently.
 *       </ul>
 *   <li>For each protector, stored under the corresponding protector ID:
 *       <ul>
 *         <li>SP_BLOB_NAME: The encrypted SP secret (the SP itself or the P0 value). Always exists.
 *         <li>PASSWORD_DATA_NAME: Data used for LSKF verification, such as the scrypt salt and
 *             parameters. Only exists for LSKF-based protectors. Doesn't exist when the LSKF is
 *             empty, except in old protectors.
 *         <li>PASSWORD_METRICS_NAME: Metrics about the LSKF, encrypted by a key derived from the
 *             SP. Only exists for LSKF-based protectors. Doesn't exist when the LSKF is empty,
 *             except in old protectors.
 *         <li>SECDISCARDABLE_NAME: A large number of random bytes that all need to be known in
 *             order to decrypt SP_BLOB_NAME. When the protector is deleted, this file is
 *             overwritten and deleted as a "best-effort" attempt to support secure deletion when
 *             hardware support for secure deletion is unavailable. Doesn't exist for LSKF-based
 *             protectors that use Weaver.
 *         <li>WEAVER_SLOT_NAME: Contains the Weaver slot number used by this protector. Only exists
 *             if the protector uses Weaver.
 *         <li>FAILURE_COUNTER_NAME: Contains the failure counter for the software rate-limiter.
 *             Only exists for LSKF-based protectors. Does not affect the hardware rate-limiter
 *             which operates concurrently.
 *       </ul>
 * </ul>
 */
class SyntheticPasswordManager {
    private static final String SP_BLOB_NAME = "spblob";
@@ -140,8 +146,8 @@ class SyntheticPasswordManager {
    private static final String WEAVER_SLOT_NAME = "weaver";
    private static final String PASSWORD_METRICS_NAME = "metrics";
    private static final String VENDOR_AUTH_SECRET_NAME = "vendor_auth_secret";
    @VisibleForTesting static final String WRONG_GUESS_COUNTER_NAME = "wrong_guess_counter";
    @VisibleForTesting static final int WRONG_GUESS_COUNTER_FILE_SIZE = 2 * Integer.BYTES;
    @VisibleForTesting static final String FAILURE_COUNTER_NAME = "failure_counter";
    @VisibleForTesting static final int FAILURE_COUNTER_FILE_SIZE = 2 * Integer.BYTES;

    // used for files associated with the SP itself, not with a particular protector
    public static final long NULL_PROTECTOR_ID = 0L;
@@ -2149,10 +2155,10 @@ class SyntheticPasswordManager {
        return (int) (n * 0x1E35A7BD);
    }

    /** Reads a wrong guess counter. */
    public int readWrongGuessCounter(LskfIdentifier id) {
        byte[] bytes = loadState(WRONG_GUESS_COUNTER_NAME, id.protectorId, id.userId);
        if (bytes == null || bytes.length != WRONG_GUESS_COUNTER_FILE_SIZE) {
    /** Reads a failure counter. */
    public int readFailureCounter(LskfIdentifier id) {
        byte[] bytes = loadState(FAILURE_COUNTER_NAME, id.protectorId, id.userId);
        if (bytes == null || bytes.length != FAILURE_COUNTER_FILE_SIZE) {
            // For a new protector, the counter is initially zero. Handle that by just returning
            // zero when the counter file does not exist. Of course, attackers who can delete the
            // file can reset the counter, but that is out of scope of the threat model of the
@@ -2168,7 +2174,7 @@ class SyntheticPasswordManager {
            Slogf.e(
                    TAG,
                    "%s file is corrupted! counter=%d, hash=0x%x",
                    WRONG_GUESS_COUNTER_NAME,
                    FAILURE_COUNTER_NAME,
                    counter,
                    hash);
            // Gracefully recover from a corrupted counter file by returning zero. This means that
@@ -2179,15 +2185,15 @@ class SyntheticPasswordManager {
        return counter;
    }

    /** Synchronously writes a wrong guess counter. */
    public void writeWrongGuessCounter(LskfIdentifier id, int count) {
        ByteBuffer buffer = ByteBuffer.allocate(WRONG_GUESS_COUNTER_FILE_SIZE);
    /** Synchronously writes a failure counter. */
    public void writeFailureCounter(LskfIdentifier id, int count) {
        ByteBuffer buffer = ByteBuffer.allocate(FAILURE_COUNTER_FILE_SIZE);
        buffer.putInt(count);
        // Add redundancy by also storing a hash of the counter. This makes it possible to
        // gracefully recover from file corruption such as bitflips, which otherwise could cause a
        // device lockout by making it seem like a lot of guesses have been made.
        buffer.putInt(hashInt(count));
        saveState(WRONG_GUESS_COUNTER_NAME, buffer.array(), id.protectorId, id.userId);
        saveState(FAILURE_COUNTER_NAME, buffer.array(), id.protectorId, id.userId);
        syncState(id.userId);
    }

+2 −2
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ public class LockSettingsServiceSwRateLimiterNotEnforcingTests
        final LskfIdentifier lskfId = new LskfIdentifier(userId, protectorId);

        // The software and hardware counters start at 0.
        assertEquals(0, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(0, mSpManager.readFailureCounter(lskfId));
        assertEquals(0, mSpManager.getSumOfWeaverFailureCounters());

        // Try the same wrong PIN repeatedly.
@@ -79,7 +79,7 @@ public class LockSettingsServiceSwRateLimiterNotEnforcingTests
        // The software counter should now be 1, since there was one unique guess. The hardware
        // counter should now be numGuesses, since the software rate-limiter was in non-enforcing
        // mode, i.e. the hardware rate-limiter should still have been called for every guess.
        assertEquals(1, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(1, mSpManager.readFailureCounter(lskfId));
        assertEquals(numGuesses, mSpManager.getSumOfWeaverFailureCounters());
    }
}
+4 −4
Original line number Diff line number Diff line
@@ -652,7 +652,7 @@ public class LockSettingsServiceTests extends BaseLockSettingsServiceTests {
        final LskfIdentifier lskfId = new LskfIdentifier(userId, protectorId);

        // The software and hardware counters start at 0.
        assertEquals(0, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(0, mSpManager.readFailureCounter(lskfId));
        assertEquals(0, mSpManager.getSumOfWeaverFailureCounters());

        // Try the same wrong PIN repeatedly.
@@ -668,7 +668,7 @@ public class LockSettingsServiceTests extends BaseLockSettingsServiceTests {
            assertEquals(0, response.getTimeout());
        }
        // The software and hardware counters should now be 1, for 1 unique guess.
        assertEquals(1, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(1, mSpManager.readFailureCounter(lskfId));
        assertEquals(1, mSpManager.getSumOfWeaverFailureCounters());
    }

@@ -686,7 +686,7 @@ public class LockSettingsServiceTests extends BaseLockSettingsServiceTests {
        final LskfIdentifier lskfId = new LskfIdentifier(userId, protectorId);

        // The software and hardware counters start at 0.
        assertEquals(0, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(0, mSpManager.readFailureCounter(lskfId));
        assertEquals(0, mSpManager.getSumOfWeaverFailureCounters());

        // Try the same wrong PIN repeatedly.
@@ -699,7 +699,7 @@ public class LockSettingsServiceTests extends BaseLockSettingsServiceTests {
        // The software counter should still be 0, since the software rate-limiter is fully disabled
        // and thus it should have never been told about the guesses at all. The hardware counter
        // should now be numGuesses, as all the (duplicate) guesses should have been sent to it.
        assertEquals(0, mSpManager.readWrongGuessCounter(lskfId));
        assertEquals(0, mSpManager.readFailureCounter(lskfId));
        assertEquals(numGuesses, mSpManager.getSumOfWeaverFailureCounters());
    }

Loading