diff --git a/app/ui/legacy/src/main/java/com/fsck/k9/view/RecipientSelectView.java b/app/ui/legacy/src/main/java/com/fsck/k9/view/RecipientSelectView.java
index 72e46e091e5ca21ac2d40830578294f379b8d4a9..9a160a5edc3bcf729fb8442fdaf8e9358b336f5c 100644
--- a/app/ui/legacy/src/main/java/com/fsck/k9/view/RecipientSelectView.java
+++ b/app/ui/legacy/src/main/java/com/fsck/k9/view/RecipientSelectView.java
@@ -404,6 +404,7 @@ public class RecipientSelectView extends TokenCompleteTextView<Recipient> implem
 
         switch (loader.getId()) {
             case LOADER_ID_FILTERING: {
+                data.removeAll(getObjects());
                 adapter.setRecipients(data);
                 break;
             }
@@ -459,6 +460,9 @@ public class RecipientSelectView extends TokenCompleteTextView<Recipient> implem
         alternatesPopup.dismiss();
 
         List<Recipient> currentRecipients = getObjects();
+        if (currentRecipients.contains(alternateAddress)) {
+            return;
+        }
         int indexOfRecipient = currentRecipients.indexOf(recipientToReplace);
         if (indexOfRecipient == -1) {
             Timber.e("Tried to refresh invalid view token!");