PARTIALLY REMOVING MAXAGE (was: [PATCH v4] make maxage use UIDs to avoid timezone issues)
Janna Martl
janna.martl109 at gmail.com
Tue Mar 31 08:53:29 UTC 2015
remove IMAP-IMAP support for maxage; add startdate, folder_startdate to compensate
maxage was fundamentally wrong in the IMAP-IMAP case: it assumed that
remote messages have UIDs in the same order as their local counterparts,
which could be false, e.g. when messages are copied in quick succession.
So, remove support for maxage in the IMAP-IMAP case.
Instead, support a special case of date restriction in IMAP-IMAP syncs:
suppose there's a pre-existing IMAP folder that you want to sync to a
new, empty folder, but you only want to sync the messages after a
certain date. This date can be specified by the configuration option
folder_startdate. On all subsequent syncs, the messages in the original
folder after folder_startdate, and all the messages in the new folder,
are considered.
Also add the option startdate, which is similar to maxage but involves
syncing messages after a fixed date, instead of a fixed number of days
ago. This is not supported in the IMAP-IMAP case for the same reasons as
maxage.
---
offlineimap.conf | 43 +++++++++-
offlineimap/accounts.py | 187 ++++++++++++++++++++++++++++++------------
offlineimap/folder/Base.py | 28 +++++++
offlineimap/folder/Gmail.py | 4 +-
offlineimap/folder/IMAP.py | 79 ++++++------------
offlineimap/folder/Maildir.py | 76 ++++++++---------
6 files changed, 267 insertions(+), 150 deletions(-)
diff --git a/offlineimap.conf b/offlineimap.conf
index af382fb..c7751b2 100644
--- a/offlineimap.conf
+++ b/offlineimap.conf
@@ -260,7 +260,7 @@ remoterepository = RemoteExample
# This option stands in the [Account Test] section.
#
# OfflineImap can replace a number of full updates by quick synchronizations.
-# This option is ignored if maxage is used.
+# This option is ignored if maxage, startdate, or folder_startdate are used.
#
# It only synchronizes a folder if
#
@@ -342,6 +342,9 @@ remoterepository = RemoteExample
#
# Known edge cases are described in the offlineimap(1).
#
+# maxage is not supported for syncing two IMAP (or Gmail) accounts, and
+# may not be used in conjunction with startdate or folder_startdate.
+#
# The maxage option expects an integer (for the number of days).
#
#maxage = 3
@@ -349,6 +352,22 @@ remoterepository = RemoteExample
# This option stands in the [Account Test] section.
#
+# When you are starting to sync an already existing account you can tell
+# OfflineIMAP to only sync messages starting at a particular date. When you do
+# this, messages older than that date will be completely ignored. This can be
+# useful for importing existing accounts when you do not want to download large
+# amounts of archive email.
+#
+# startdate is not supported for syncing two IMAP (or Gmail) accounts,
+# and may not be used in conjunction with maxage or folder_startdate.
+#
+# The startdate option expects a date in the format yyyy-mm-dd.
+#
+#startdate = 2015-03-20
+
+
+# This option stands in the [Account Test] section.
+#
# Maildir file format uses colon (:) separator between uniq name and info.
# Unfortunatelly colon is not allowed character in windows file name. If you
# enable maildir-windows-compatible option, OfflineIMAP will be able to store
@@ -451,6 +470,28 @@ localfolders = ~/Test
# This option stands in the [Repository LocalExample] section.
#
+# Use this option if LocalExample is a pre-existing account that you
+# want to clone to a new, empty account, but you only want to clone
+# messages later than a particular date. If you are using this option
+# for the first time, RemoteExample must be empty. On all subsequent
+# syncs (for which this option is enabled), offlineimap will consider
+# the messages in LocalExample later than folder_startdate, and will
+# consider all messages in RemoteExample. Older messages in LocalExample
+# will be completely ignored.
+#
+# This option is mainly useful for IMAP-IMAP syncs, because maxage and
+# startdate are not supported in that case.
+#
+# folder_startdate may not be used in conjunction with maxage or
+# startdate.
+#
+# The folder_startdate option expects a date in the format yyyy-mm-dd.
+#
+#folder_startdate = 2015-03-20
+
+
+# This option stands in the [Repository LocalExample] section.
+#
# Some users may not want the atime (last access time) of folders to be
# modified by OfflineIMAP. If 'restoreatime' is set to yes, OfflineIMAP
# will restore the atime of the "new" and "cur" folders in each maildir
diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py
index a7ad623..e4da366 100644
--- a/offlineimap/accounts.py
+++ b/offlineimap/accounts.py
@@ -17,10 +17,11 @@
from subprocess import Popen, PIPE
from threading import Event
import os
+import time
from sys import exc_info
import traceback
-from offlineimap import mbnames, CustomConfig, OfflineImapError
+from offlineimap import mbnames, CustomConfig, OfflineImapError, imaplibutil
from offlineimap import globals
from offlineimap.repository import Repository
from offlineimap.ui import getglobalui
@@ -402,6 +403,96 @@ def syncfolder(account, remotefolder, quick):
Filtered folders on the remote side will not invoke this function."""
+
+ def sanity_check(date):
+ if date[0] < 1900:
+ raise OfflineImapError("date restriction led to year %d. "
+ "Abort syncing."% min_date[0],
+ OfflineImapError.ERROR.REPO)
+
+ def check_uid_validity(localfolder, remotefolder, statusfolder):
+ # If either the local or the status folder has messages and
+ # there is a UID validity problem, warn and abort. If there are
+ # no messages, UW IMAPd loses UIDVALIDITY. But we don't really
+ # need it if both local folders are empty. So, in that case,
+ # just save it off.
+ if localfolder.getmessagecount() or statusfolder.getmessagecount():
+ if not localfolder.check_uidvalidity():
+ ui.validityproblem(localfolder)
+ localrepos.restore_atime()
+ return
+ if not remotefolder.check_uidvalidity():
+ ui.validityproblem(remotefolder)
+ localrepos.restore_atime()
+ return
+ else:
+ # Both folders empty, just save new UIDVALIDITY
+ localfolder.save_uidvalidity()
+ remotefolder.save_uidvalidity()
+
+ def save_min_uid(folder, min_uid):
+ uidfile = folder.get_min_uid_file()
+ fd = open(uidfile, 'wt')
+ fd.write(str(min_uid) + "\n")
+ fd.close()
+
+ def cachemessagelists_by_date(localfolder, remotefolder, date):
+ """ Returns messages with uid > min(uids of within-date
+ messages)."""
+
+ localfolder.cachemessagelist(min_date=date)
+ check_uid_validity(localfolder, remotefolder, statusfolder)
+ # local messagelist had date restriction applied already. Restrict
+ # sync to messages with UIDs >= min_uid from this list.
+ #
+ # local messagelist might contain new messages (with uid's < 0).
+ positive_uids = filter(
+ lambda uid: uid > 0, localfolder.getmessageuidlist())
+ if len(positive_uids) > 0:
+ remotefolder.cachemessagelist(min_uid=min(positive_uids))
+ else:
+ # No messages with UID > 0 in range in localfolder.
+ # date restriction was applied with respect to local dates but
+ # remote folder timezone might be different from local, so be
+ # safe and make sure the range isn't bigger than in local.
+ remotefolder.cachemessagelist(
+ min_date=time.gmtime(time.mktime(date) + 24*60*60))
+
+ def cachemessagelists_folder_startdate(new, partial, datestr):
+ """ Retrieve messagelists when folder_startdate has been set for
+ the folder 'partial'.
+
+ Idea: suppose you want to clone the messages after date in one
+ account (partial) to a new one (new). If new is empty, then copy
+ messages in partial newer than date to new, and keep track of the
+ min uid. On subsequent syncs, sync all the messages in new against
+ those after that min uid in partial. This is a partial replacement
+ for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
+ the UIDs of the messages in localfolder might not be in the same
+ order as those of corresponding messages in remotefolder, so if L in
+ local corresponds to R in remote, the ranges [L, ...] and [R, ...]
+ might not correspond. But, if we're cloning a folder into a new one,
+ [min_uid, ...] does correspond to [1, ...].
+
+ This is just for IMAP-IMAP. For Maildir-IMAP, use startdate instead.
+ """
+
+ new.cachemessagelist()
+ if not new.getmessageuidlist():
+ # First sync
+ date = time.strptime(datestr, "%Y-%m-%d")
+ partial.cachemessagelist(min_date=date)
+ uids = partial.getmessageuidlist()
+ if len(uids) > 0:
+ min_uid = min(uids)
+ else:
+ min_uid = 1
+ save_min_uid(partial, min_uid)
+ else:
+ min_uid = partial.get_min_uid()
+ partial.cachemessagelist(min_uid=min_uid)
+
+
remoterepos = account.remoterepos
localrepos = account.localrepos
statusrepos = account.statusrepos
@@ -429,68 +520,56 @@ def syncfolder(account, remotefolder, quick):
statusfolder.cachemessagelist()
- maxage = localfolder.getmaxage()
# Load local folder.
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
- ui.loadmessagelist(localrepos, localfolder)
- # For maxage, returns messages with uid > min(uids of within-maxage messages).
- localfolder.cachemessagelist(maxage=maxage)
- ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount())
-
- if quick:
- # IMAP quickchanged isn't compatible with maxage, since the "quick"
- # check can only retrieve a full list of UIDs in the folder.
- if maxage:
- ui.warn("Quick syncs (-q) not supported in conjunction "\
- "with maxage; ignoring -q.")
- else:
+
+ #Retrieve messagelists, taking into account age-restriction
+ #options
+ maxage = localfolder.getmaxage()
+ localstart = localfolder.getfolder_startdate()
+ remotestart = remotefolder.getfolder_startdate()
+ startdate = localfolder.getstartdate()
+ if (maxage != None) + (localstart != None) + (remotestart != None)\
+ + (startdate != None) > 1:
+ raise OfflineImapError("You can set at most one of the "
+ "following: maxage, folder_startdate (for the local folder), "
+ "folder_startdate (for the remote folder), startdate",
+ OfflineImapError.ERROR.REPO), None, exc_info()[2]
+ if (maxage != None or localstart or remotestart or startdate)\
+ and quick:
+ # IMAP quickchanged isn't compatible with options that
+ # involve restricting the messagelist, since the "quick"
+ # check can only retrieve a full list of UIDs in the folder.
+ ui.warn("Quick syncs (-q) not supported in conjunction "
+ "with maxage, folder_startdate, or startdate; ignoring "
+ "-q.")
+ if maxage != None:
+ timelimit = time.gmtime(time.time() - 60*60*24*maxage)
+ sanity_check(timelimit)
+ cachemessagelists_by_date(localfolder, remotefolder, timelimit)
+ elif startdate != None:
+ timelimit = time.strptime(startdate, "%Y-%m-%d")
+ sanity_check(timelimit)
+ cachemessagelists_by_date(localfolder, remotefolder, timelimit)
+ elif localstart != None:
+ cachemessagelists_folder_startdate(remotefolder, localfolder,
+ localstart)
+ check_uid_validity(localfolder, remotefolder, statusfolder)
+ elif remotestart != None:
+ cachemessagelists_folder_startdate(localfolder, remotefolder,
+ remotestart)
+ check_uid_validity(localfolder, remotefolder, statusfolder)
+ else:
+ localfolder.cachemessagelist()
+ if quick:
if (not localfolder.quickchanged(statusfolder) and
not remotefolder.quickchanged(statusfolder)):
ui.skippingfolder(remotefolder)
localrepos.restore_atime()
return
-
- # If either the local or the status folder has messages and
- # there is a UID validity problem, warn and abort. If there are
- # no messages, UW IMAPd loses UIDVALIDITY. But we don't really
- # need it if both local folders are empty. So, in that case,
- # just save it off.
- if localfolder.getmessagecount() or statusfolder.getmessagecount():
- if not localfolder.check_uidvalidity():
- ui.validityproblem(localfolder)
- localrepos.restore_atime()
- return
- if not remotefolder.check_uidvalidity():
- ui.validityproblem(remotefolder)
- localrepos.restore_atime()
- return
- else:
- # Both folders empty, just save new UIDVALIDITY
- localfolder.save_uidvalidity()
- remotefolder.save_uidvalidity()
-
- # Load remote folder.
- ui.loadmessagelist(remoterepos, remotefolder)
- if maxage != None:
- # local messagelist was applied maxage, already. Restrict sync for
- # messages with UIDs >= min_uid from this list.
- #
- # local messagelist might contain new messages (with uid's < 0).
- positive_uids = filter(
- lambda uid: uid > 0, localfolder.getmessageuidlist())
- if len(positive_uids) > 0:
- remotefolder.cachemessagelist(min_uid=min(positive_uids))
- else:
- # No messages with UID > 0 in range in localfolder.
- # maxage was applied with respect to local dates but
- # remote folder timezone might be different from local, so be
- # safe and make sure the range isn't bigger than in local.
- remotefolder.cachemessagelist(maxage=maxage - 1)
- else:
+ check_uid_validity(localfolder, remotefolder, statusfolder)
remotefolder.cachemessagelist()
- ui.messagelistloaded(remoterepos, remotefolder,
- remotefolder.getmessagecount())
# Synchronize remote changes.
if not localrepos.getconfboolean('readonly', False):
diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py
index 4af0361..4dbcebd 100644
--- a/offlineimap/folder/Base.py
+++ b/offlineimap/folder/Base.py
@@ -306,6 +306,34 @@ class BaseFolder(object):
return self.config.getdefaultint("Account %s"%
self.accountname, "maxsize", None)
+ def getstartdate(self):
+ return self.config.getdefault("Account %s"%
+ self.accountname, "startdate", None)
+
+ def getfolder_startdate(self):
+ """ Retrieve the value of the configuration option folder_startdate """
+ return self.config.getdefault("Repository " + self.repository.name,
+ 'folder_startdate', None)
+
+ def get_min_uid_file(self):
+ startuiddir = os.path.join(self.config.getmetadatadir(),
+ 'Repository-' + self.repository.name, 'StartUID')
+ if not os.path.exists(startuiddir):
+ os.mkdir(startuiddir, 0o700)
+ return os.path.join(startuiddir, self.getfolderbasename())
+
+ def get_min_uid(self):
+ uidfile = self.get_min_uid_file()
+ try:
+ fd = open(uidfile, 'rt')
+ min_uid = long(fd.readline().strip())
+ fd.close()
+ return min_uid
+ except:
+ raise IOError("Can't read %s. To start using folder_startdate, "\
+ "folder must be empty"% uidfile)
+
+
def savemessage(self, uid, content, flags, rtime):
"""Writes a new message, with the specified uid.
diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py
index 93e8eee..7f239a3 100644
--- a/offlineimap/folder/Gmail.py
+++ b/offlineimap/folder/Gmail.py
@@ -121,9 +121,9 @@ class GmailFolder(IMAPFolder):
# TODO: merge this code with the parent's cachemessagelist:
# TODO: they have too much common logics.
- def cachemessagelist(self, maxage=None, min_uid=None):
+ def cachemessagelist(self, min_date=None, min_uid=None):
if not self.synclabels:
- return super(GmailFolder, self).cachemessagelist(maxage=maxage,
+ return super(GmailFolder, self).cachemessagelist(min_date=min_date,
min_uid=min_uid)
self.messagelist = {}
diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py
index 25dd26d..cabf5b9 100644
--- a/offlineimap/folder/IMAP.py
+++ b/offlineimap/folder/IMAP.py
@@ -18,6 +18,7 @@
import random
import binascii
import re
+import os
import time
from sys import exc_info
@@ -79,6 +80,18 @@ class IMAPFolder(BaseFolder):
def waitforthread(self):
self.imapserver.connectionwait()
+ def getmaxage(self):
+ if self.config.getdefaultint("Account %s"%
+ self.accountname, "maxage", None):
+ raise OfflineImapError("maxage is not supported on IMAP-IMAP sync",
+ OfflineImapError.ERROR.REPO), None, exc_info()[2]
+
+ def getstartdate(self):
+ if self.config.getdefault("Account %s"%
+ self.accountname, "startdate", None):
+ raise OfflineImapError("startdate is not supported on IMAP-IMAP sync",
+ OfflineImapError.ERROR.REPO), None, exc_info()[2]
+
# Interface from BaseFolder
def getcopyinstancelimit(self):
return 'MSGCOPY_' + self.repository.getname()
@@ -143,8 +156,7 @@ class IMAPFolder(BaseFolder):
return True
return False
-
- def _msgs_to_fetch(self, imapobj, maxage=None, min_uid=None):
+ def _msgs_to_fetch(self, imapobj, min_date=None, min_uid=None):
"""Determines sequence numbers of messages to be fetched.
Message sequence numbers (MSNs) are more easily compacted
@@ -152,10 +164,10 @@ class IMAPFolder(BaseFolder):
Arguments:
- imapobj: instance of IMAPlib
- - maxage (optional): only fetch messages up to maxage days ago
+ - min_date (optional): a time_struct; only fetch messages newer than this
- min_uid (optional): only fetch messages with UID >= min_uid
- This function should be called with at MOST one of maxage OR
+ This function should be called with at MOST one of min_date OR
min_uid set but not BOTH.
Returns: range(s) for messages or None if no messages
@@ -183,60 +195,20 @@ class IMAPFolder(BaseFolder):
# 1. min_uid condition.
if min_uid != None:
conditions.append("UID %d:*"% min_uid)
- # 2. maxage condition.
- elif maxage != None:
+ # 2. date condition.
+ elif min_date != None:
# Find out what the oldest message is that we should look at.
- # FIXME: we are checking maxage validity way too late. Also, we are
- # doing similar computing in MaildirFolder()._iswithinmaxage()
- # WITHOUT this sanity check. We really want to work with
- # oldest_struct as soon as possible.
- oldest_struct = time.gmtime(time.time() - (60*60*24*maxage))
- if oldest_struct[0] < 1900:
- raise OfflineImapError("maxage setting led to year %d. "
- "Abort syncing."% oldest_struct[0],
- OfflineImapError.ERROR.REPO)
conditions.append("SINCE %02d-%s-%d"% (
- oldest_struct[2],
- MonthNames[oldest_struct[1]],
- oldest_struct[0]))
+ min_date[2], MonthNames[min_date[1]], min_date[0]))
# 3. maxsize condition.
maxsize = self.getmaxsize()
if maxsize != None:
conditions.append("SMALLER %d"% maxsize)
- if len(conditions) > 1:
+ if len(conditions) >= 1:
# Build SEARCH command.
- if maxage == None:
- search_cond = "(%s)"% ' '.join(conditions)
- search_result = search(search_cond)
- else:
- # Get the messages within maxage is not enough. We want all
- # messages with UID > min_uid from these within-maxage messages.
- # We can't rely on maxage only to get the proper min_uid because
- # the internal date used by the SINCE command might sightly
- # diverge from the date and time the message was assigned its
- # UID. Same logic as applied for the Maildir, we have to
- # re-include some messages.
- #
- # Ordering by UID is the same as ordering by MSN, so we get the
- # messages with MSN > min_msn of the within-maxage messages.
- msg_seq_numbers = map(lambda s : int(s), search_result)
- if len(msg_seq_numbers) < 1:
- return None # Nothing to sync.
- min_msn = min(msg_seq_numbers)
- # If no maxsize, can just ask for all messages with MSN > min_msn.
- search_cond = "%d:*"% min_msn
- if maxsize != None:
- # Restrict the range min_msn:* to those with acceptable size.
- # Single-quotes prevent imaplib2 from quoting the sequence.
- search_cond = "'%s (SMALLER %d)'"% (min_msn, maxsize)
- # Having to make a second query sucks but this is only for
- # IMAP/IMAP configurations with maxage enabled. We assume this
- # is not so common. This time overhead should be acceptable
- # regarding the benefits introduced by all the avoided sync of
- # maxage.
- search_result = search(search_cond)
- # Resulting MSN are separated by space, coalesce into ranges
+ search_cond = "(%s)"% ' '.join(conditions)
+ search_result = search(search_cond)
return imaputil.uid_sequence(search_result)
# By default consider all messages in this folder.
@@ -248,12 +220,14 @@ class IMAPFolder(BaseFolder):
# Interface from BaseFolder
- def cachemessagelist(self, maxage=None, min_uid=None):
+ def cachemessagelist(self, min_date=None, min_uid=None):
+ self.ui.loadmessagelist(self.repository, self)
self.messagelist = {}
imapobj = self.imapserver.acquireconnection()
try:
- msgsToFetch = self._msgs_to_fetch(imapobj, maxage=maxage, min_uid=min_uid)
+ msgsToFetch = self._msgs_to_fetch(
+ imapobj, min_date=min_date, min_uid=min_uid)
if not msgsToFetch:
return # No messages to sync
@@ -285,6 +259,7 @@ class IMAPFolder(BaseFolder):
flags = imaputil.flagsimap2maildir(options['FLAGS'])
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
+ self.ui.messagelistloaded(self.repository, self, self.getmessagecount())
def dropmessagelistcache(self):
self.messagelist = {}
diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index 49ce82b..d400a3f 100644
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -91,25 +91,17 @@ class MaildirFolder(BaseFolder):
token."""
return 42
- # Checks to see if the given message is within the maximum age according
- # to the maildir name which should begin with a timestamp
- def _iswithinmaxage(self, messagename, maxage):
- # In order to have similar behaviour as SINCE in an IMAP search
- # we must convert this to the oldest time and then strip off hrs/mins
- # from that day.
- oldest_time_utc = time.time() - (60*60*24*maxage)
- oldest_time_struct = time.gmtime(oldest_time_utc)
- oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \
- + (oldest_time_struct[4] * 60) \
- + oldest_time_struct[5])
- oldest_time_utc -= oldest_time_today_seconds
+ def _iswithintime(self, messagename, date):
+ """Check to see if the given message is newer than date (a
+ time_struct) according to the maildir name which should begin
+ with a timestamp."""
timestampmatch = re_timestampmatch.search(messagename)
if not timestampmatch:
return True
timestampstr = timestampmatch.group()
timestamplong = long(timestampstr)
- if(timestamplong < oldest_time_utc):
+ if(timestamplong < time.mktime(date)):
return False
else:
return True
@@ -150,13 +142,13 @@ class MaildirFolder(BaseFolder):
flags = set((c for c in flagmatch.group(1) if not c.islower()))
return prefix, uid, fmd5, flags
- def _scanfolder(self, maxage=None):
+ def _scanfolder(self, min_date=None, min_uid=None):
"""Cache the message list from a Maildir.
- If using maxage, this finds the min UID of all messages within maxage
- and use it as the real cursor up to where we go back in time. This handles
- the edge cases where the date is much earlier than messages with similar
- UID's (e.g. the UID was reassigned much later).
+ If min_date is set, this finds the min UID of all messages newer than
+ min_date and uses it as the real cutoff for considering messages.
+ This handles the edge cases where the date is much earlier than messages
+ with similar UID's (e.g. the UID was reassigned much later).
Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F
(flagged).
@@ -173,7 +165,7 @@ class MaildirFolder(BaseFolder):
files.extend((dirannex, filename) for
filename in os.listdir(fulldirname))
- maxage_excludees = {}
+ date_excludees = {}
for dirannex, filename in files:
# We store just dirannex and filename, ie 'cur/123...'
filepath = os.path.join(dirannex, filename)
@@ -194,34 +186,36 @@ class MaildirFolder(BaseFolder):
nouidcounter -= 1
else:
uid = long(uidmatch.group(1))
- if maxage != None and not self._iswithinmaxage(filename, maxage):
- # Keep track of messages outside of maxage, because they still
- # might have UID > min(UIDs of within-maxage). We hit this case
- # if any message had a known/valid datetime and was re-uploaded
- # because the UID in the filename got lost (e.g. local
- # copy/move). On next sync, it was assigned a new UID from the
- # server and will be included in the SEARCH condition. So, we
- # must re-include them later in this method in order to avoid
- # inconsistent lists of messages.
- maxage_excludees[uid] = self.msglist_item_initializer(uid)
- maxage_excludees[uid]['flags'] = flags
- maxage_excludees[uid]['filename'] = filepath
+ if min_uid != None and uid > 0 and uid < min_uid:
+ continue
+ if min_date != None and not self._iswithintime(filename, min_date):
+ # Keep track of messages outside of the time limit, because they
+ # still might have UID > min(UIDs of within-min_date). We hit
+ # this case for maxage if any message had a known/valid datetime
+ # and was re-uploaded because the UID in the filename got lost
+ # (e.g. local copy/move). On next sync, it was assigned a new
+ # UID from the server and will be included in the SEARCH
+ # condition. So, we must re-include them later in this method
+ # in order to avoid inconsistent lists of messages.
+ date_excludees[uid] = self.msglist_item_initializer(uid)
+ date_excludees[uid]['flags'] = flags
+ date_excludees[uid]['filename'] = filepath
else:
# 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S
retval[uid] = self.msglist_item_initializer(uid)
retval[uid]['flags'] = flags
retval[uid]['filename'] = filepath
- if maxage != None:
+ if min_date != None:
# Re-include messages with high enough uid's.
positive_uids = filter(lambda uid: uid > 0, retval)
if positive_uids:
min_uid = min(positive_uids)
- for uid in maxage_excludees.keys():
+ for uid in date_excludees.keys():
if uid > min_uid:
# This message was originally excluded because of
- # maxage. It is re-included now because we want all
+ # its date. It is re-included now because we want all
# messages with UID > min_uid.
- retval[uid] = maxage_excludees[uid]
+ retval[uid] = date_excludees[uid]
return retval
# Interface from BaseFolder
@@ -245,9 +239,12 @@ class MaildirFolder(BaseFolder):
return {'flags': set(), 'filename': '/no-dir/no-such-file/'}
# Interface from BaseFolder
- def cachemessagelist(self, maxage=None):
+ def cachemessagelist(self, min_date=None, min_uid=None):
if self.ismessagelistempty():
- self.messagelist = self._scanfolder(maxage=maxage)
+ self.ui.loadmessagelist(self.repository, self)
+ self.messagelist = self._scanfolder(min_date=min_date,
+ min_uid=min_uid)
+ self.ui.messagelistloaded(self.repository, self, self.getmessagecount())
# Interface from BaseFolder
def getmessagelist(self):
@@ -448,10 +445,7 @@ class MaildirFolder(BaseFolder):
os.unlink(filepath)
except OSError:
# Can't find the file -- maybe already deleted?
- maxage = self.getmaxage()
- # get more than enough messages to be sure we captured the min UID
- # expected on the other side
- newmsglist = self._scanfolder(maxage=maxage + 1)
+ newmsglist = self._scanfolder()
if uid in newmsglist: # Nope, try new filename.
filename = newmsglist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
--
2.3.4
More information about the OfflineIMAP-project
mailing list