Use UidSet

inbox/mailsync/backends/gmail.py

@@ -20,7 +20,9 @@
 
 """
 
+import itertools
 from collections import OrderedDict
+from collections.abc import Iterable
 from datetime import datetime, timedelta
 from threading import Semaphore
 from typing import TYPE_CHECKING, Dict, List
@@ -38,6 +40,7 @@
 from inbox.models.backends.imap import ImapFolderInfo, ImapThread, ImapUid
 from inbox.models.category import EPOCH
 from inbox.models.session import session_scope
+from inbox.uid_set import UidSet
 from inbox.util.debug import bind_context
 from inbox.util.itert import chunk
 
@@ -277,6 +280,7 @@ def initial_sync_impl(self, crispin_client: "CrispinClient") -> None:
                     change_poller, "changepoller", self.account_id, self.folder_id
                 )
 
+                uids_to_download: Iterable[int]
                 if self.is_all_mail(crispin_client):
                     # Prioritize UIDs for messages in the inbox folder.
                     if len_remote_uids < 1e6:
@@ -294,17 +298,18 @@ def initial_sync_impl(self, crispin_client: "CrispinClient") -> None:
                             )
                         )
 
-                    uids_to_download = sorted(unknown_uids - inbox_uids) + sorted(
-                        unknown_uids & inbox_uids
+                    uids_to_download = itertools.chain(
+                        reversed(UidSet(unknown_uids & inbox_uids)),
+                        reversed(UidSet(unknown_uids - inbox_uids)),
                     )
 
                     del inbox_uids  # free up memory as soon as possible
                 else:
-                    uids_to_download = sorted(unknown_uids)
+                    uids_to_download = reversed(UidSet(unknown_uids))
 
                 del unknown_uids  # free up memory as soon as possible
 
-            for uids in chunk(reversed(uids_to_download), 1024):
+            for uids in chunk(uids_to_download, 1024):
                 g_metadata = crispin_client.g_metadata(uids)
                 # UIDs might have been expunged since sync started, in which
                 # case the g_metadata call above will return nothing.

inbox/mailsync/backends/imap/generic.py

@@ -78,6 +78,7 @@
 from inbox.interruptible_threading import InterruptibleThread
 from inbox.logging import get_logger
 from inbox.util.concurrency import introduce_jitter, retry_with_logging
+from inbox.uid_set import UidSet
 from inbox.util.debug import bind_context
 from inbox.util.itert import chunk
 from inbox.util.misc import or_none
@@ -470,9 +471,10 @@ def initial_sync_impl(self, crispin_client: CrispinClient):
                         local_uids.difference(remote_uids),
                     )
 
-                new_uids = sorted(remote_uids.difference(local_uids), reverse=True)
+                new_uids = UidSet(remote_uids.difference(local_uids))
 
                 len_remote_uids = len(remote_uids)
+                del local_uids  # free up memory as soon as possible
                 del remote_uids  # free up memory as soon as possible
                 del local_uids  # free up memory as soon as possible
 
@@ -489,7 +491,7 @@ def initial_sync_impl(self, crispin_client: CrispinClient):
             change_poller = ChangePoller(self)
             change_poller.start()
             bind_context(change_poller, "changepoller", self.account_id, self.folder_id)
-            for count, uid in enumerate(new_uids, start=1):
+            for count, uid in enumerate(reversed(new_uids), start=1):
                 # The speedup from batching appears to be less clear for
                 # non-Gmail accounts, so for now just download one-at-a-time.
                 self.download_and_commit_uids(crispin_client, [uid])
@@ -799,9 +801,12 @@ def get_new_uids(self, crispin_client):
                 self.account_id, db_session, self.folder_id
             )
         latest_uids = crispin_client.conn.fetch(f"{lastseenuid + 1}:*", ["UID"]).keys()
-        new_uids = set(latest_uids) - {lastseenuid}
+        new_uids = UidSet(set(latest_uids) - {lastseenuid})
+
+        del latest_uids  # free up memory as soon as possible
+
         if new_uids:
-            for uid in sorted(new_uids):
+            for uid in new_uids:
                 self.download_and_commit_uids(crispin_client, [uid])
         self.uidnext = remote_uidnext