From 2179c10892d606b7a4bab7180b0ed71de497c0a2 Mon Sep 17 00:00:00 2001 From: vincent Bourgmayer Date: Tue, 4 Oct 2022 11:06:00 +0200 Subject: [PATCH 1/6] Progressively delay sync in case of failure --- .../foundation/e/drive/EdriveApplication.java | 14 +--- .../database/FailedSyncPrefsManager.java | 72 +++++++++++++++++++ .../e/drive/services/ResetService.java | 3 +- .../services/SynchronizationService.java | 45 ++++++++---- .../e/drive/utils/AppConstants.java | 1 - 5 files changed, 106 insertions(+), 29 deletions(-) create mode 100644 app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java diff --git a/app/src/main/java/foundation/e/drive/EdriveApplication.java b/app/src/main/java/foundation/e/drive/EdriveApplication.java index 0e440fa8..6e80e804 100644 --- a/app/src/main/java/foundation/e/drive/EdriveApplication.java +++ b/app/src/main/java/foundation/e/drive/EdriveApplication.java @@ -19,6 +19,7 @@ import android.util.Log; import foundation.e.drive.FileObservers.FileEventListener; import foundation.e.drive.FileObservers.RecursiveFileObserver; +import foundation.e.drive.database.FailedSyncPrefsManager; import foundation.e.drive.services.SynchronizationService; import foundation.e.drive.utils.AppConstants; import foundation.e.drive.utils.CommonUtils; @@ -50,7 +51,7 @@ public class EdriveApplication extends Application { Log.d(TAG, "Account already registered"); startRecursiveFileObserver(); - resetTransferFailureCounter(); + FailedSyncPrefsManager.getInstance(getApplicationContext()).clearPreferences(); final Intent SynchronizationServiceIntent = new Intent(getApplicationContext(), SynchronizationService.class); startService(SynchronizationServiceIntent); @@ -87,17 +88,6 @@ public class EdriveApplication extends Application { Log.d(TAG, "RecursiveFileObserver on root folder stops watching "); } - - /** - * Clear sharedPreference that store failed transfer counter. - * Doing this here, allow to restart it once a day. - */ - private void resetTransferFailureCounter() { - final SharedPreferences prefs = getSharedPreferences(AppConstants.FAILED_TRANSFER_PREF, Context.MODE_PRIVATE); - prefs.edit().clear().commit(); - } - - @Override public void onLowMemory() { super.onLowMemory(); diff --git a/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java new file mode 100644 index 00000000..36f59a30 --- /dev/null +++ b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java @@ -0,0 +1,72 @@ +/* + * Copyright © ECORP SAS 2022. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0 + * which accompanies this distribution, and is available at + * http://www.gnu.org/licenses/gpl.html + */ + +package foundation.e.drive.database; + +import android.content.Context; +import android.content.SharedPreferences; + +/** + * @author vincent Bourgmayer + */ +public class FailedSyncPrefsManager { + public final static String PREF_NAME = "failed_transfer"; + private final static String FAILURE_COUNTER_KEY = "C"; + private final static String FAILURE_TIME_KEY ="T"; + + private static FailedSyncPrefsManager instance = null; + + private SharedPreferences preferences; + private FailedSyncPrefsManager() {}; + + public static FailedSyncPrefsManager getInstance(Context context) { + if (instance == null) { + instance = new FailedSyncPrefsManager(); + instance.preferences = context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE); + } + + return instance; + } + + + public int getFailureCounterForFile(int fileStateId) { + return preferences.getInt(getFailureCounterKey(fileStateId), 0); + } + + public long getLastFailureTimeForFile(int fileStateId) { + return preferences.getLong(getFailureTimeKey(fileStateId), 0); + } + + public void saveFileSyncFailure(int fileStateId) { + final int failure_counter = getFailureCounterForFile(fileStateId)+1; + final long timestamp = System.currentTimeMillis()/1000; + + preferences.edit() + .putInt(getFailureCounterKey(fileStateId), failure_counter) + .putLong(getFailureTimeKey(fileStateId), timestamp) + .commit(); + } + + public void clearPreferences() { + preferences.edit().clear().commit(); + } + + public void removeDataForFile(int fileStateId) { + preferences.edit() + .remove(getFailureCounterKey(fileStateId)) + .remove(getFailureTimeKey(fileStateId)) + .commit(); + } + private String getFailureCounterKey(int fileStateId) { + return fileStateId + FAILURE_COUNTER_KEY; + } + + private String getFailureTimeKey(int fileStateId) { + return fileStateId + FAILURE_TIME_KEY; + } +} diff --git a/app/src/main/java/foundation/e/drive/services/ResetService.java b/app/src/main/java/foundation/e/drive/services/ResetService.java index da07a532..47a5e4a7 100644 --- a/app/src/main/java/foundation/e/drive/services/ResetService.java +++ b/app/src/main/java/foundation/e/drive/services/ResetService.java @@ -21,6 +21,7 @@ import java.io.File; import foundation.e.drive.EdriveApplication; import foundation.e.drive.database.DbHelper; +import foundation.e.drive.database.FailedSyncPrefsManager; import foundation.e.drive.utils.AppConstants; import static foundation.e.drive.utils.AppConstants.INITIALFOLDERS_NUMBER; @@ -93,7 +94,7 @@ public class ResetService extends Service { .apply(); } - deleteSharedPreferences(AppConstants.FAILED_TRANSFER_PREF); + deleteSharedPreferences(FailedSyncPrefsManager.PREF_NAME); } private void removeCachedFiles() { diff --git a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java index 363e8167..d4aafdae 100644 --- a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java +++ b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java @@ -7,8 +7,6 @@ */ package foundation.e.drive.services; -import static foundation.e.drive.utils.AppConstants.FAILED_TRANSFER_PREF; - import android.accounts.Account; import android.accounts.AccountManager; import android.app.Service; @@ -34,6 +32,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import foundation.e.drive.database.DbHelper; +import foundation.e.drive.database.FailedSyncPrefsManager; import foundation.e.drive.models.SyncRequest; import foundation.e.drive.models.SyncWrapper; import foundation.e.drive.operations.DownloadFileOperation; @@ -112,10 +111,7 @@ public class SynchronizationService extends Service implements OnRemoteOperation return; } } - final SharedPreferences prefs = getSharedPreferences(FAILED_TRANSFER_PREF, Context.MODE_PRIVATE); - if (prefs.getInt(request.getSyncedFileState().getLocalPath(), 0) >= FAILURE_LIMIT) { - return; - } + if (skipBecauseOfPreviousFail(request)) return; syncRequestQueue.remove(request); syncRequestQueue.add(request); @@ -135,13 +131,34 @@ public class SynchronizationService extends Service implements OnRemoteOperation requests.removeIf(syncRequest -> syncWrapper.equals(syncRequest)); } } - final SharedPreferences prefs = getSharedPreferences(FAILED_TRANSFER_PREF, Context.MODE_PRIVATE); - requests.removeIf(syncRequest -> prefs.getInt(syncRequest.getSyncedFileState().getLocalPath(), 0) >= FAILURE_LIMIT); + + requests.removeIf(syncRequest -> skipBecauseOfPreviousFail(syncRequest)); syncRequestQueue.removeAll(requests); syncRequestQueue.addAll(requests); } + /** + * Progressively delay synchronization of a file in case of failure + * @param request SyncRequest for the given file + * @return false if file can be sync or true if it must wait + */ + private boolean skipBecauseOfPreviousFail(SyncRequest request) { + final FailedSyncPrefsManager failedSyncPref = FailedSyncPrefsManager.getInstance(getApplicationContext()); + final int fileStateId = request.getSyncedFileState().getId(); + final int failureCounter = failedSyncPref.getFailureCounterForFile(fileStateId); + + if (failureCounter == 0) return false; + if (failureCounter > FAILURE_LIMIT) return true; + + long delay = 3600; //1h + if (failureCounter == 2) delay = 7200; //2h + else if (failureCounter == 3) delay = 18000; //5h + + final long timeStamp = System.currentTimeMillis()/1000; + return timeStamp < failedSyncPref.getLastFailureTimeForFile(fileStateId) + delay; + } + public void startSynchronization(){ Log.d(TAG, "startAllThreads"); for(int i =-1; ++i < workerAmount;){ @@ -253,16 +270,14 @@ public class SynchronizationService extends Service implements OnRemoteOperation } private void updateFailureCounter(SyncRequest request, boolean success) { - final SharedPreferences prefs = getSharedPreferences(FAILED_TRANSFER_PREF, Context.MODE_PRIVATE); - final String failure_key = request.getSyncedFileState().getLocalPath(); - final int previous_failure_count = prefs.getInt(failure_key, 0); - final SharedPreferences.Editor pref_editor = prefs.edit(); + final FailedSyncPrefsManager failedPref = FailedSyncPrefsManager.getInstance(getApplicationContext()); + final int fileStateId = request.getSyncedFileState().getId(); + if (success) { - pref_editor.remove(failure_key); + failedPref.removeDataForFile(fileStateId); } else { - pref_editor.putInt(failure_key, previous_failure_count + 1); + failedPref.saveFileSyncFailure(fileStateId); } - pref_editor.commit(); } public class SynchronizationBinder extends Binder{ diff --git a/app/src/main/java/foundation/e/drive/utils/AppConstants.java b/app/src/main/java/foundation/e/drive/utils/AppConstants.java index 441924e3..69be96f2 100644 --- a/app/src/main/java/foundation/e/drive/utils/AppConstants.java +++ b/app/src/main/java/foundation/e/drive/utils/AppConstants.java @@ -40,7 +40,6 @@ public abstract class AppConstants { public static final String ACCOUNT_DATA_ALIAS_KEY = "alias"; public static final String ACCOUNT_DATA_EMAIL = "email"; public static final String ACCOUNT_USER_ID_KEY = "USERID"; - public final static String FAILED_TRANSFER_PREF = "failed_transfer"; public static final String[] MEDIA_SYNCABLE_CATEGORIES = new String[]{"Images", "Movies", "Music", "Ringtones", "Documents", "Podcasts"}; public static final String[] SETTINGS_SYNCABLE_CATEGORIES = new String[]{"Rom settings"}; -- GitLab From a128f54934d81dd6c090367c15b6eef85d825145 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Tue, 4 Oct 2022 10:14:03 +0000 Subject: [PATCH 2/6] Apply 1 suggestion(s) to 1 file(s) --- .../foundation/e/drive/database/FailedSyncPrefsManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java index 36f59a30..35333694 100644 --- a/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java +++ b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java @@ -44,7 +44,7 @@ public class FailedSyncPrefsManager { public void saveFileSyncFailure(int fileStateId) { final int failure_counter = getFailureCounterForFile(fileStateId)+1; - final long timestamp = System.currentTimeMillis()/1000; + final long timestamp = System.currentTimeMillis() / 1000; preferences.edit() .putInt(getFailureCounterKey(fileStateId), failure_counter) -- GitLab From d247791ecc4b0e621defb6f85b97df2fd65cd016 Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Tue, 4 Oct 2022 10:14:11 +0000 Subject: [PATCH 3/6] Apply 1 suggestion(s) to 1 file(s) --- .../foundation/e/drive/database/FailedSyncPrefsManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java index 35333694..e901661e 100644 --- a/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java +++ b/app/src/main/java/foundation/e/drive/database/FailedSyncPrefsManager.java @@ -17,7 +17,7 @@ import android.content.SharedPreferences; public class FailedSyncPrefsManager { public final static String PREF_NAME = "failed_transfer"; private final static String FAILURE_COUNTER_KEY = "C"; - private final static String FAILURE_TIME_KEY ="T"; + private final static String FAILURE_TIME_KEY = "T"; private static FailedSyncPrefsManager instance = null; -- GitLab From 9a70e42e3306be5d3702b5d7e010925b62c4b0cc Mon Sep 17 00:00:00 2001 From: Abhishek Aggarwal Date: Tue, 4 Oct 2022 10:26:27 +0000 Subject: [PATCH 4/6] Apply 1 suggestion(s) to 1 file(s) --- .../foundation/e/drive/services/SynchronizationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java index d4aafdae..1e5383e9 100644 --- a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java +++ b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java @@ -155,7 +155,7 @@ public class SynchronizationService extends Service implements OnRemoteOperation if (failureCounter == 2) delay = 7200; //2h else if (failureCounter == 3) delay = 18000; //5h - final long timeStamp = System.currentTimeMillis()/1000; + final long timeStamp = System.currentTimeMillis() / 1000; return timeStamp < failedSyncPref.getLastFailureTimeForFile(fileStateId) + delay; } -- GitLab From 0f97df61e36834b2e7976d83390c3674034d9937 Mon Sep 17 00:00:00 2001 From: vincent Bourgmayer Date: Tue, 4 Oct 2022 12:33:03 +0200 Subject: [PATCH 5/6] apply Abhishek suggestion: replace if by switch --- .../services/SynchronizationService.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java index 1e5383e9..d04d3e85 100644 --- a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java +++ b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java @@ -48,7 +48,6 @@ import foundation.e.drive.utils.DavClientProvider; public class SynchronizationService extends Service implements OnRemoteOperationListener { private final static String TAG = SynchronizationService.class.getSimpleName(); private final SynchronizationBinder binder = new SynchronizationBinder(); - private final static int FAILURE_LIMIT = 3; private ConcurrentLinkedDeque syncRequestQueue; private ConcurrentHashMap startedSync; //Integer is thread index (1 to workerAmount) @@ -148,12 +147,22 @@ public class SynchronizationService extends Service implements OnRemoteOperation final int fileStateId = request.getSyncedFileState().getId(); final int failureCounter = failedSyncPref.getFailureCounterForFile(fileStateId); - if (failureCounter == 0) return false; - if (failureCounter > FAILURE_LIMIT) return true; - - long delay = 3600; //1h - if (failureCounter == 2) delay = 7200; //2h - else if (failureCounter == 3) delay = 18000; //5h + long delay; + switch (failureCounter) { + case 0: + return false; + case 1: + delay = 3600; //1h + break; + case 2: + delay = 7200; //2h + break; + case 3: + delay = 18000; // 5h + break; + default: + return true; + } final long timeStamp = System.currentTimeMillis() / 1000; return timeStamp < failedSyncPref.getLastFailureTimeForFile(fileStateId) + delay; -- GitLab From c43fa20a0c618470f6880f15d2df2a9005b395fc Mon Sep 17 00:00:00 2001 From: vincent Bourgmayer Date: Tue, 4 Oct 2022 15:35:41 +0200 Subject: [PATCH 6/6] do not delay chunked upload in case of failure --- .../e/drive/operations/UploadFileOperation.java | 2 +- .../e/drive/services/SynchronizationService.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/foundation/e/drive/operations/UploadFileOperation.java b/app/src/main/java/foundation/e/drive/operations/UploadFileOperation.java index 2c6bdb09..96e36f48 100644 --- a/app/src/main/java/foundation/e/drive/operations/UploadFileOperation.java +++ b/app/src/main/java/foundation/e/drive/operations/UploadFileOperation.java @@ -42,7 +42,7 @@ import foundation.e.drive.utils.DavClientProvider; */ public class UploadFileOperation extends RemoteOperation { private final static String TAG = UploadFileOperation.class.getSimpleName(); - private final static int FILE_SIZE_FLOOR_FOR_CHUNKED = 3072000; //3MB + public final static int FILE_SIZE_FLOOR_FOR_CHUNKED = 3072000; //3MB private final Context context; private final SyncedFileState syncedState; private final Account account; // TODO Remove as soon as nextcloud library move all Operation to NextcloudClient instead of OwncloudClient diff --git a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java index d04d3e85..73c95b30 100644 --- a/app/src/main/java/foundation/e/drive/services/SynchronizationService.java +++ b/app/src/main/java/foundation/e/drive/services/SynchronizationService.java @@ -7,6 +7,8 @@ */ package foundation.e.drive.services; +import static foundation.e.drive.operations.UploadFileOperation.FILE_SIZE_FLOOR_FOR_CHUNKED; + import android.accounts.Account; import android.accounts.AccountManager; import android.app.Service; @@ -26,6 +28,7 @@ import com.owncloud.android.lib.common.operations.OnRemoteOperationListener; import com.owncloud.android.lib.common.operations.RemoteOperation; import com.owncloud.android.lib.common.operations.RemoteOperationResult; +import java.io.File; import java.util.Collection; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -285,6 +288,12 @@ public class SynchronizationService extends Service implements OnRemoteOperation if (success) { failedPref.removeDataForFile(fileStateId); } else { + if (request.getOperationType().equals(SyncRequest.Type.UPLOAD)) { + final File file = new File(request.getSyncedFileState().getLocalPath()); + if (file.length() >= FILE_SIZE_FLOOR_FOR_CHUNKED) { + return; //do not delay future synchronization when it's for a chunked upload + } + } failedPref.saveFileSyncFailure(fileStateId); } } -- GitLab