Merge pull request #5598 from k9mail/move_LocalStore_compact (b73659c1) · Commits · e / os / Mail

app/core/src/main/java/com/fsck/k9/controller/MessagingController.java

+17 −71

Original line number	Diff line number	Diff line
		@@ -289,18 +289,13 @@ public class MessagingController {
		return localStore.getFolderServerId(folderId);
		}

		private long getFolderId(Account account, String folderServerId) throws MessagingException {
		LocalStore localStore = getLocalStoreOrThrow(account);
		return localStore.getFolderId(folderServerId);
		}

		private long getFolderIdOrThrow(Account account, String folderServerId) {
		LocalStore localStore = getLocalStoreOrThrow(account);
		try {
		return localStore.getFolderId(folderServerId);
		} catch (MessagingException e) {
		throw new IllegalStateException(e);
		private long getFolderId(Account account, String folderServerId) {
		MessageStore messageStore = messageStoreManager.getMessageStore(account);
		Long folderId = messageStore.getFolderId(folderServerId);
		if (folderId == null) {
		throw new IllegalStateException("Folder not found (server ID: " + folderServerId + ")");
		}
		return folderId;
		}

		public void addListener(MessagingListener listener) {
		@@ -604,7 +599,7 @@ public class MessagingController {
		);
		}

		public void synchronizeMailboxBlocking(Account account, String folderServerId) throws MessagingException {
		public void synchronizeMailboxBlocking(Account account, String folderServerId) {
		long folderId = getFolderId(account, folderServerId);

		account.setRingNotified(false);
		@@ -2509,16 +2504,13 @@ public class MessagingController {
		@Override
		public void run() {
		try {
		LocalStore localStore = localStoreProvider.getInstance(account);
		long oldSize = localStore.getSize();
		localStore.compact();
		long newSize = localStore.getSize();
		MessageStore messageStore = messageStoreManager.getMessageStore(account);
		long oldSize = messageStore.getSize();
		messageStore.compact();
		long newSize = messageStore.getSize();
		for (MessagingListener l : getListeners(ml)) {
		l.accountSizeChanged(account, oldSize, newSize);
		}
		} catch (UnavailableStorageException e) {
		Timber.i("Failed to compact account because storage is not available - trying again later.");
		throw new UnavailableAccountException(e);
		} catch (Exception e) {
		Timber.e(e, "Failed to compact account %s", account.getDescription());
		}
		@@ -2526,52 +2518,6 @@ public class MessagingController {
		});
		}

		public void clear(final Account account, final MessagingListener ml) {
		putBackground("clear:" + account.getDescription(), ml, new Runnable() {
		@Override
		public void run() {
		try {
		LocalStore localStore = localStoreProvider.getInstance(account);
		long oldSize = localStore.getSize();
		localStore.clear();
		localStore.resetVisibleLimits(account.getDisplayCount());
		long newSize = localStore.getSize();
		for (MessagingListener l : getListeners(ml)) {
		l.accountSizeChanged(account, oldSize, newSize);
		}
		} catch (UnavailableStorageException e) {
		Timber.i("Failed to clear account because storage is not available - trying again later.");
		throw new UnavailableAccountException(e);
		} catch (Exception e) {
		Timber.e(e, "Failed to clear account %s", account.getDescription());
		}
		}
		});
		}

		public void recreate(final Account account, final MessagingListener ml) {
		putBackground("recreate:" + account.getDescription(), ml, new Runnable() {
		@Override
		public void run() {
		try {
		LocalStore localStore = localStoreProvider.getInstance(account);
		long oldSize = localStore.getSize();
		localStore.recreate();
		localStore.resetVisibleLimits(account.getDisplayCount());
		long newSize = localStore.getSize();
		for (MessagingListener l : getListeners(ml)) {
		l.accountSizeChanged(account, oldSize, newSize);
		}
		} catch (UnavailableStorageException e) {
		Timber.i("Failed to recreate an account because storage is not available - trying again later.");
		throw new UnavailableAccountException(e);
		} catch (Exception e) {
		Timber.e(e, "Failed to recreate account %s", account.getDescription());
		}
		}
		});
		}

		public void deleteAccount(Account account) {
		notificationController.clearNewMailNotifications(account);
		memorizingMessagingListener.removeAccount(account);
		@@ -2734,7 +2680,7 @@ public class MessagingController {

		@Override
		public void syncStarted(@NotNull String folderServerId) {
		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		for (MessagingListener messagingListener : getListeners(listener)) {
		messagingListener.synchronizeMailboxStarted(account, folderId);
		}
		@@ -2770,7 +2716,7 @@ public class MessagingController {

		@Override
		public void syncProgress(@NotNull String folderServerId, int completed, int total) {
		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		for (MessagingListener messagingListener : getListeners(listener)) {
		messagingListener.synchronizeMailboxProgress(account, folderId, completed, total);
		}
		@@ -2804,7 +2750,7 @@ public class MessagingController {
		}

		String accountUuid = account.getUuid();
		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		MessageReference messageReference = new MessageReference(accountUuid, folderId, messageServerId, null);
		notificationController.removeNewMailNotification(account, messageReference);
		}
		@@ -2831,7 +2777,7 @@ public class MessagingController {

		@Override
		public void syncFinished(@NotNull String folderServerId) {
		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		for (MessagingListener messagingListener : getListeners(listener)) {
		messagingListener.synchronizeMailboxFinished(account, folderId);
		}
		@@ -2847,7 +2793,7 @@ public class MessagingController {
		notifyUserIfCertificateProblem(account, exception, true);
		}

		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		for (MessagingListener messagingListener : getListeners(listener)) {
		messagingListener.synchronizeMailboxFailed(account, folderId, message);
		}
		@@ -2855,7 +2801,7 @@ public class MessagingController {

		@Override
		public void folderStatusChanged(@NotNull String folderServerId) {
		long folderId = getFolderIdOrThrow(account, folderServerId);
		long folderId = getFolderId(account, folderServerId);
		for (MessagingListener messagingListener : getListeners(listener)) {
		messagingListener.folderStatusChanged(account, folderId);
		}

app/core/src/main/java/com/fsck/k9/mailstore/LocalStore.java

+0 −191

Original line number	Diff line number	Diff line
		@@ -30,7 +30,6 @@ import android.text.TextUtils;
		import com.fsck.k9.Account;
		import com.fsck.k9.Clock;
		import com.fsck.k9.DI;
		import com.fsck.k9.K9;
		import com.fsck.k9.controller.MessageCounts;
		import com.fsck.k9.Preferences;
		import com.fsck.k9.controller.MessagingControllerCommands.PendingCommand;
		@@ -53,10 +52,7 @@ import com.fsck.k9.mailstore.LockableDatabase.SchemaDefinition;
		import com.fsck.k9.mailstore.LockableDatabase.WrappedException;
		import com.fsck.k9.mailstore.StorageManager.InternalStorageProvider;
		import com.fsck.k9.mailstore.StorageManager.StorageProvider;
		import com.fsck.k9.message.extractors.AttachmentCounter;
		import com.fsck.k9.message.extractors.AttachmentInfoExtractor;
		import com.fsck.k9.message.extractors.MessageFulltextCreator;
		import com.fsck.k9.message.extractors.MessagePreviewCreator;
		import com.fsck.k9.provider.EmailProvider;
		import com.fsck.k9.provider.EmailProvider.MessageColumns;
		import com.fsck.k9.search.LocalSearch;
		@@ -170,9 +166,6 @@ public class LocalStore {

		private final Context context;
		private final ContentResolver contentResolver;
		private final MessagePreviewCreator messagePreviewCreator;
		private final MessageFulltextCreator messageFulltextCreator;
		private final AttachmentCounter attachmentCounter;
		private final PendingCommandSerializer pendingCommandSerializer;
		private final AttachmentInfoExtractor attachmentInfoExtractor;

		@@ -193,9 +186,6 @@ public class LocalStore {
		this.context = context;
		this.contentResolver = context.getContentResolver();

		messagePreviewCreator = MessagePreviewCreator.newInstance();
		messageFulltextCreator = MessageFulltextCreator.newInstance();
		attachmentCounter = AttachmentCounter.newInstance();
		pendingCommandSerializer = PendingCommandSerializer.getInstance();
		attachmentInfoExtractor = DI.get(AttachmentInfoExtractor.class);

		@@ -244,122 +234,6 @@ public class LocalStore {
		return outboxStateRepository;
		}

		public long getSize() throws MessagingException {

		final StorageManager storageManager = StorageManager.getInstance(context);

		final File attachmentDirectory = storageManager.getAttachmentDirectory(account.getUuid(),
		database.getStorageProviderId());

		return database.execute(false, new DbCallback<Long>() {
		@Override
		public Long doDbWork(final SQLiteDatabase db) {
		final File[] files = attachmentDirectory.listFiles();
		long attachmentLength = 0;
		if (files != null) {
		for (File file : files) {
		if (file.exists()) {
		attachmentLength += file.length();
		}
		}
		}

		final File dbFile = storageManager.getDatabase(account.getUuid(), database.getStorageProviderId());
		return dbFile.length() + attachmentLength;
		}
		});
		}

		public void compact() throws MessagingException {
		if (K9.isDebugLoggingEnabled()) {
		Timber.i("Before compaction size = %d", getSize());
		}

		database.execute(false, new DbCallback<Void>() {
		@Override
		public Void doDbWork(final SQLiteDatabase db) throws WrappedException {
		db.execSQL("VACUUM");
		return null;
		}
		});

		if (K9.isDebugLoggingEnabled()) {
		Timber.i("After compaction size = %d", getSize());
		}
		}


		public void clear() throws MessagingException {
		if (K9.isDebugLoggingEnabled()) {
		Timber.i("Before prune size = %d", getSize());
		}

		deleteAllMessageDataFromDisk();

		if (K9.isDebugLoggingEnabled()) {
		Timber.i("After prune / before compaction size = %d", getSize());
		Timber.i("Before clear folder count = %d", getFolderCount());
		Timber.i("Before clear message count = %d", getMessageCount());
		Timber.i("After prune / before clear size = %d", getSize());
		}

		database.execute(false, new DbCallback<Void>() {
		@Override
		public Void doDbWork(final SQLiteDatabase db) {
		// We don't care about threads of deleted messages, so delete the whole table.
		db.delete("threads", null, null);

		// Don't delete deleted messages. They are essentially placeholders for UIDs of messages that have
		// been deleted locally.
		db.delete("messages", "deleted = 0", null);

		// We don't need the search data now either
		db.delete("messages_fulltext", null, null);

		return null;
		}
		});

		compact();

		if (K9.isDebugLoggingEnabled()) {
		Timber.i("After clear message count = %d", getMessageCount());
		Timber.i("After clear size = %d", getSize());
		}
		}

		private int getMessageCount() throws MessagingException {
		return database.execute(false, new DbCallback<Integer>() {
		@Override
		public Integer doDbWork(final SQLiteDatabase db) {
		Cursor cursor = null;
		try {
		cursor = db.rawQuery("SELECT COUNT(*) FROM messages", null);
		cursor.moveToFirst();
		return cursor.getInt(0); // message count
		} finally {
		Utility.closeQuietly(cursor);
		}
		}
		});
		}

		private int getFolderCount() throws MessagingException {
		return database.execute(false, new DbCallback<Integer>() {
		@Override
		public Integer doDbWork(final SQLiteDatabase db) {
		Cursor cursor = null;
		try {
		cursor = db.rawQuery("SELECT COUNT(*) FROM folders", null);
		cursor.moveToFirst();
		return cursor.getInt(0); // folder count
		} finally {
		Utility.closeQuietly(cursor);
		}
		}
		});
		}

		public LocalFolder getFolder(String serverId) {
		return new LocalFolder(this, serverId);
		}
		@@ -412,44 +286,6 @@ public class LocalStore {
		database.delete();
		}

		public void recreate() throws UnavailableStorageException {
		database.recreate();
		}

		private void deleteAllMessageDataFromDisk() throws MessagingException {
		markAllMessagePartsDataAsMissing();
		deleteAllMessagePartsDataFromDisk();
		}

		private void markAllMessagePartsDataAsMissing() throws MessagingException {
		database.execute(false, new DbCallback<Void>() {
		@Override
		public Void doDbWork(final SQLiteDatabase db) throws WrappedException {
		ContentValues cv = new ContentValues();
		cv.put("data_location", DataLocation.MISSING);
		db.update("message_parts", cv, null, null);

		return null;
		}
		});
		}

		private void deleteAllMessagePartsDataFromDisk() {
		final StorageManager storageManager = StorageManager.getInstance(context);
		File attachmentDirectory = storageManager.getAttachmentDirectory(
		account.getUuid(), database.getStorageProviderId());
		File[] files = attachmentDirectory.listFiles();
		if (files == null) {
		return;
		}

		for (File file : files) {
		if (file.exists() && !file.delete()) {
		file.deleteOnExit();
		}
		}
		}

		public void resetVisibleLimits(int visibleLimit) throws MessagingException {
		final ContentValues cv = new ContentValues();
		cv.put("visible_limit", Integer.toString(visibleLimit));
		@@ -875,21 +711,6 @@ public class LocalStore {
		});
		}

		public long getFolderId(String folderServerId) throws MessagingException {
		return database.execute(false, db -> {
		try (Cursor cursor = db.query("folders", new String[] { "id" },
		"server_id = ?", new String[] { folderServerId },
		null, null, null)
		) {
		if (cursor.moveToFirst()) {
		return cursor.getLong(0);
		} else {
		throw new MessagingException("Folder not found by server ID: " + folderServerId);
		}
		}
		});
		}

		public static class AttachmentInfo {
		public String name;
		public long size;
		@@ -937,18 +758,6 @@ public class LocalStore {
		return database;
		}

		MessagePreviewCreator getMessagePreviewCreator() {
		return messagePreviewCreator;
		}

		public MessageFulltextCreator getMessageFulltextCreator() {
		return messageFulltextCreator;
		}

		AttachmentCounter getAttachmentCounter() {
		return attachmentCounter;
		}

		AttachmentInfoExtractor getAttachmentInfoExtractor() {
		return attachmentInfoExtractor;
		}

app/core/src/main/java/com/fsck/k9/mailstore/MessageStore.kt

+10 −0

Original line number	Diff line number	Diff line
		@@ -125,6 +125,11 @@ interface MessageStore {
		*/
		fun getLastUid(folderId: Long): Long?

		/**
		* Return the size of this message store in bytes.
		*/
		fun getSize(): Long

		/**
		* Remove messages from the store.
		*/
		@@ -267,4 +272,9 @@ interface MessageStore {
		* Create or update a number property associated with the given folder.
		*/
		fun setFolderExtraNumber(folderId: Long, name: String, value: Long)

		/**
		* Optimize the message store with the goal of using the minimal amount of disk space.
		*/
		fun compact()
		}

app/storage/src/main/java/com/fsck/k9/storage/messages/DatabaseOperations.kt

0 → 100644

+40 −0

Original line number	Diff line number	Diff line
		package com.fsck.k9.storage.messages

		import com.fsck.k9.mailstore.LockableDatabase
		import com.fsck.k9.mailstore.StorageManager
		import timber.log.Timber

		internal class DatabaseOperations(
		private val lockableDatabase: LockableDatabase,
		val storageManager: StorageManager,
		val accountUuid: String
		) {
		fun getSize(): Long {
		val storageProviderId = lockableDatabase.storageProviderId
		val attachmentDirectory = storageManager.getAttachmentDirectory(accountUuid, storageProviderId)

		return lockableDatabase.execute(false) {
		val attachmentFiles = attachmentDirectory.listFiles() ?: emptyArray()
		val attachmentsSize = attachmentFiles.asSequence()
		.filter { file -> file.exists() }
		.fold(initial = 0L) { accumulatedSize, file ->
		accumulatedSize + file.length()
		}

		val databaseFile = storageManager.getDatabase(accountUuid, storageProviderId)
		val databaseSize = databaseFile.length()

		databaseSize + attachmentsSize
		}
		}

		fun compact() {
		Timber.i("Before compaction size = %d", getSize())

		lockableDatabase.execute(false) { database ->
		database.execSQL("VACUUM")
		}

		Timber.i("After compaction size = %d", getSize())
		}
		}

app/storage/src/main/java/com/fsck/k9/storage/messages/K9MessageStore.kt

+9 −0

Original line number	Diff line number	Diff line
		@@ -40,6 +40,7 @@ class K9MessageStore(
		private val updateFolderOperations = UpdateFolderOperations(database)
		private val deleteFolderOperations = DeleteFolderOperations(database, attachmentFileManager)
		private val keyValueStoreOperations = KeyValueStoreOperations(database)
		private val databaseOperations = DatabaseOperations(database, storageManager, accountUuid)

		override fun saveRemoteMessage(folderId: Long, messageServerId: String, messageData: SaveMessageData) {
		saveMessageOperations.saveRemoteMessage(folderId, messageServerId, messageData)
		@@ -133,6 +134,10 @@ class K9MessageStore(
		return retrieveFolderOperations.getFolderId(folderServerId)
		}

		override fun getSize(): Long {
		return databaseOperations.getSize()
		}

		override fun changeFolder(folderServerId: String, name: String, type: FolderType) {
		updateFolderOperations.changeFolder(folderServerId, name, type)
		}
		@@ -208,4 +213,8 @@ class K9MessageStore(
		override fun setFolderExtraNumber(folderId: Long, name: String, value: Long) {
		return keyValueStoreOperations.setFolderExtraNumber(folderId, name, value)
		}

		override fun compact() {
		return databaseOperations.compact()
		}
		}