[PATCH 3/4] IMAP: revamp _msgs_to_fetch()

[Janna Martl]

On Wed, Mar 25, 2015 at 05:42:29AM +0100, Nicolas Sebrecht wrote:
>Signed-off-by: Nicolas Sebrecht <nicolas.s-dev at laposte.net>
>---
> offlineimap/folder/IMAP.py | 96 +++++++++++++---------------------------------
> 1 file changed, 27 insertions(+), 69 deletions(-)
>
>diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py
>index bf61ebe..834b446 100644
>--- a/offlineimap/folder/IMAP.py
>+++ b/offlineimap/folder/IMAP.py
>@@ -153,94 +153,52 @@ class IMAPFolder(BaseFolder):
>         Arguments:
>         - imapobj: instance of IMAPlib
>         - maxage (optional): only fetch messages up to maxage days ago
>-        - min_uid (optional): only fetch messages with uid > min_uid
>+        - min_uid (optional): only fetch messages with UID >= min_uid
>
>-        This function should be called with at most one of maxage or
>-        min_uid.
>-
>-        If using maxage, this finds the min uid of all messages within
>-        maxage, and returns all messages with uid > this min. As maxage
>-        is only set if this folder is the local folder, this ensures
>-        consistency with remote (which only restricts to uid > this min)
>-        in edge cases where the date is much earlier than messages with
>-        similar UID's (e.g. the UID was reassigned much later).
>+        This function should be called with at MOST one of maxage OR
>+        min_uid set but not BOTH.
>
>         Returns: range(s) for messages or None if no messages
>         are to be fetched."""
>
>         res_type, imapdata = imapobj.select(self.getfullname(), True, True)
>         if imapdata == [None] or imapdata[0] == '0':
>-            # Empty folder, no need to populate message list
>+            # Empty folder, no need to populate message list.
>             return None
>
>-        # By default examine all messages in this folder
>-        msgsToFetch = '1:*'
>+        conditions = []
>+
>+        if min_uid != None:
>+            conditions.append("UID %d:*"% min_uid)
>+        elif maxage != None:
>+            # Find out what the oldest message is that we should look at.
>+            oldest_struct = time.gmtime(time.time() - (60*60*24*maxage))
>+            if oldest_struct[0] < 1900:
>+                raise OfflineImapError("maxage setting led to year %d. "
>+                    "Abort syncing."% oldest_struct[0],
>+                    OfflineImapError.ERROR.REPO)
>+            conditions.append("SINCE %02d-%s-%d"%
>+                oldest_struct[2],
>+                MonthNames[oldest_struct[1]],
>+                oldest_struct[0])
>
>         maxsize = self.getmaxsize()
>+        if maxsize != None:
>+            conditions.append("SMALLER %d"% maxsize)
>
>-        # Build search condition
>-        if maxage or min_uid != None or maxsize != None:
>-            search_cond = "(";
>-
>-            if maxage:
>-                #find out what the oldest message is that we should look at
>-                oldest_struct = time.gmtime(time.time() - (60*60*24*maxage))
>-                if oldest_struct[0] < 1900:
>-                    raise OfflineImapError("maxage setting led to year %d. "
>-                        "Abort syncing."% oldest_struct[0],
>-                        OfflineImapError.ERROR.REPO)
>-                search_cond += "SINCE %02d-%s-%d"% (
>-                    oldest_struct[2],
>-                    MonthNames[oldest_struct[1]],
>-                    oldest_struct[0])
>-            if min_uid != None:
>-                if maxage: # There are two conditions, add space
>-                    search_cond += " "
>-                search_cond += "UID %d:*"% min_uid
>-
>-            if maxsize != None:
>-                if maxage or min_uid != None:
>-                    # There are at least two conditions, add space
>-                    search_cond += " "
>-                search_cond += "SMALLER %d"% maxsize
>-
>-            search_cond += ")"
>-
>+        if len(conditions) > 1:
>+            # Build SEARCH conditions.
>+            search_cond = "(%s)"% ' '.join(conditions)
>             res_type, res_data = imapobj.search(None, search_cond)
>             if res_type != 'OK':
>                 raise OfflineImapError("SEARCH in folder [%s]%s failed. "
>                     "Search string was '%s'. Server responded '[%s] %s'"% (
>                     self.getrepository(), self, search_cond, res_type, res_data),
>                     OfflineImapError.ERROR.FOLDER)
>-
>-            if maxage:
>-                # Found messages within maxage, but want all messages with UID
>-                # > the min uid of the within-maxage messages. Ordering by UID
>-                # is the same as ordering by MSN, so we get the messages with
>-                # MSN > the min MSN of the within-maxage messages.
>-                msg_seq_numbers = map(lambda s : int(s), res_data[0].split())
>-                if msg_seq_numbers:
>-                    min_msn = min(msg_seq_numbers)
>-                else:
>-                    return None
>-                if maxsize == None:
>-                    # If no maxsize, can just ask for all messages with MSN > min_msn
>-                    return "%d:*"% min_msn
>-                else:
>-                    # Restrict the range min_msn:* to those with acceptable size.
>-                    # Single-quotes prevent imaplib2 from quoting the sequence.
>-                    search_cond = "'%d:* (SMALLER %d)'"% (min_msn, maxsize)
>-                    res_type, res_data = imapobj.search(None, search_cond)
>-                    if res_type != 'OK':
>-                        raise OfflineImapError("SEARCH in folder [%s]%s failed. "
>-                            "Search string was '%s'. Server responded '[%s] %s'"%
>-                            (self.getrepository(), self, search_cond, res_type,
>-                            res_data), OfflineImapError.ERROR.FOLDER)

I think this is necessary if we're doing an IMAP-IMAP sync, and right
now self is the local folder. Per our most recent discussion, for the
local folder, there are two steps: (1) find messages within maxage;
(2) expand that list to also include all messages with UID > min(uid's
within maxage). We've already done step (1) above; this is step (2),
except it looks a little different -- step (1) gave us a list of
MSN's, not UID's, so you can ask for msn's in the range min_msn:* if
there are no other conditions to enforce but if you also have to
restrict that to < maxsize, I think you need another query.

>             # Resulting MSN are separated by space, coalesce into ranges
>-            msgsToFetch = imaputil.uid_sequence(res_data[0].split())
>-
>-        return msgsToFetch
>+            return imaputil.uid_sequence(res_data[0].split())
>+        # By default examine all messages in this folder.
>+        return '1:*'
>
>     # Interface from BaseFolder
>     def msglist_item_initializer(self, uid):
>-- 
>2.3.1
>