[PATCH 01/13] Make LocalStatus use sqlite
Sebastian Spaeth
Sebastian at SSpaeth.de
Wed Nov 24 09:56:38 UTC 2010
From: Rob Browning <rlb at defaultvalue.org>
Original patch by Stewart Smith.
Signed-off-by: Sebastian Spaeth <Sebastian at SSpaeth.de>
---
offlineimap/accounts.py | 6 +-
offlineimap/folder/Base.py | 32 +++-
offlineimap/folder/IMAP.py | 2 +-
offlineimap/folder/LocalStatus.py | 151 +++++++++++------
patch | 342 +++++++++++++++++++++++++++++++++++++
5 files changed, 470 insertions(+), 63 deletions(-)
create mode 100644 patch
diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py
index 5975332..e64965b 100644
--- a/offlineimap/accounts.py
+++ b/offlineimap/accounts.py
@@ -316,13 +316,13 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
ui.loadmessagelist(localrepos, localfolder)
localfolder.cachemessagelist()
- ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys()))
+ ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount())
# If either the local or the status folder has messages and there is a UID
# validity problem, warn and abort. If there are no messages, UW IMAPd
# loses UIDVALIDITY. But we don't really need it if both local folders are
# empty. So, in that case, just save it off.
- if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()):
+ if localfolder.getmessagecount() or statusfolder.getmessagecount():
if not localfolder.isuidvalidityok():
ui.validityproblem(localfolder)
localrepos.restore_atime()
@@ -339,7 +339,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
ui.loadmessagelist(remoterepos, remotefolder)
remotefolder.cachemessagelist()
ui.messagelistloaded(remoterepos, remotefolder,
- len(remotefolder.getmessagelist().keys()))
+ remotefolder.getmessagecount())
#
diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py
index 0f38bc3..35ab450 100644
--- a/offlineimap/folder/Base.py
+++ b/offlineimap/folder/Base.py
@@ -130,6 +130,24 @@ class BaseFolder:
You must call cachemessagelist() before calling this function!"""
raise NotImplementedException
+ def uidexists(self,uid):
+ """Returns true if uid exists"""
+ mlist = self.getmessagelist()
+ if uid in mlist:
+ return 1
+ else:
+ return 0
+ return 0
+
+ def getmessageuidlist(self):
+ """Gets a list of UIDs.
+ You may have to call cachemessagelist() before calling this function!"""
+ return self.getmessagelist().keys()
+
+ def getmessagecount(self):
+ """Gets the number of messages."""
+ return len(self.getmessagelist().keys())
+
def getmessage(self, uid):
"""Returns the content of the specified message."""
raise NotImplementedException
@@ -238,7 +256,7 @@ class BaseFolder:
and once that succeeds, get the UID, add it to the others for real,
add it to local for real, and delete the fake one."""
- uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
+ uidlist = [uid for uid in self.getmessageuidlist() if uid < 0]
threads = []
usethread = None
@@ -300,11 +318,10 @@ class BaseFolder:
them to dest."""
threads = []
- dest_messagelist = dest.getmessagelist()
- for uid in self.getmessagelist().keys():
+ for uid in self.getmessageuidlist():
if uid < 0: # Ignore messages that pass 1 missed.
continue
- if not uid in dest_messagelist:
+ if not dest.uidexists(uid):
if self.suggeststhreads():
self.waitforthread()
thread = InstanceLimitedThread(\
@@ -327,11 +344,10 @@ class BaseFolder:
Look for message present in dest but not in self.
If any, delete them."""
deletelist = []
- self_messagelist = self.getmessagelist()
- for uid in dest.getmessagelist().keys():
+ for uid in dest.getmessageuidlist():
if uid < 0:
continue
- if not uid in self_messagelist:
+ if not self.uidexists(uid):
deletelist.append(uid)
if len(deletelist):
UIBase.getglobalui().deletingmessages(deletelist, applyto)
@@ -354,7 +370,7 @@ class BaseFolder:
addflaglist = {}
delflaglist = {}
- for uid in self.getmessagelist().keys():
+ for uid in self.getmessageuidlist():
if uid < 0: # Ignore messages missed by pass 1
continue
selfflags = self.getmessageflags(uid)
diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py
index c90d0e5..133bdff 100644
--- a/offlineimap/folder/IMAP.py
+++ b/offlineimap/folder/IMAP.py
@@ -56,7 +56,7 @@ class IMAPFolder(BaseFolder):
return self.accountname
def suggeststhreads(self):
- return 1
+ return 0
def waitforthread(self):
self.imapserver.connectionwait()
diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py
index 157989d..0972a36 100644
--- a/offlineimap/folder/LocalStatus.py
+++ b/offlineimap/folder/LocalStatus.py
@@ -19,9 +19,17 @@
from Base import BaseFolder
import os, threading
+from pysqlite2 import dbapi2 as sqlite
+
magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1"
+newmagicline = "OFFLINEIMAP LocalStatus NOW IN SQLITE, DO NOT MODIFY"
class LocalStatusFolder(BaseFolder):
+ def __deinit__(self):
+ self.save()
+ self.cursor.close()
+ self.connection.close()
+
def __init__(self, root, name, repository, accountname, config):
self.name = name
self.root = root
@@ -30,12 +38,51 @@ class LocalStatusFolder(BaseFolder):
self.dofsync = config.getdefaultboolean("general", "fsync", True)
self.filename = os.path.join(root, name)
self.filename = repository.getfolderfilename(name)
- self.messagelist = None
+ self.messagelist = {}
self.repository = repository
self.savelock = threading.Lock()
self.doautosave = 1
self.accountname = accountname
BaseFolder.__init__(self)
+ self.dbfilename = self.filename + '.sqlite'
+
+ # MIGRATE
+ if os.path.exists(self.filename):
+ self.connection = sqlite.connect(self.dbfilename)
+ self.cursor = self.connection.cursor()
+ self.cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))')
+ if self.isnewfolder():
+ self.messagelist = {}
+ return
+ file = open(self.filename, "rt")
+ self.messagelist = {}
+ line = file.readline().strip()
+ assert(line == magicline)
+ for line in file.xreadlines():
+ line = line.strip()
+ uid, flags = line.split(':')
+ uid = long(uid)
+ flags = [x for x in flags]
+ flags.sort()
+ flags = ''.join(flags)
+ self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)',
+ (uid,flags))
+ file.close()
+ self.connection.commit()
+ os.rename(self.filename, self.filename + ".old")
+ self.cursor.close()
+ self.connection.close()
+
+ # create new
+ if not os.path.exists(self.dbfilename):
+ self.connection = sqlite.connect(self.dbfilename)
+ self.cursor = self.connection.cursor()
+ self.cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))')
+ else:
+ self.connection = sqlite.connect(self.dbfilename)
+ self.cursor = self.connection.cursor()
+
+
def getaccountname(self):
return self.accountname
@@ -44,7 +91,7 @@ class LocalStatusFolder(BaseFolder):
return 0
def isnewfolder(self):
- return not os.path.exists(self.filename)
+ return not os.path.exists(self.dbfilename)
def getname(self):
return self.name
@@ -60,84 +107,85 @@ class LocalStatusFolder(BaseFolder):
def deletemessagelist(self):
if not self.isnewfolder():
- os.unlink(self.filename)
+ self.cursor.close()
+ self.connection.close()
+ os.unlink(self.dbfilename)
def cachemessagelist(self):
- if self.isnewfolder():
- self.messagelist = {}
- return
- file = open(self.filename, "rt")
- self.messagelist = {}
- line = file.readline().strip()
- if not line and not line.read():
- # The status file is empty - should not have happened,
- # but somehow did.
- file.close()
- return
- assert(line == magicline)
- for line in file.xreadlines():
- line = line.strip()
- uid, flags = line.split(':')
- uid = long(uid)
- flags = [x for x in flags]
- self.messagelist[uid] = {'uid': uid, 'flags': flags}
- file.close()
+ return
def autosave(self):
if self.doautosave:
self.save()
def save(self):
- self.savelock.acquire()
- try:
- file = open(self.filename + ".tmp", "wt")
- file.write(magicline + "\n")
- for msg in self.messagelist.values():
- flags = msg['flags']
- flags.sort()
- flags = ''.join(flags)
- file.write("%s:%s\n" % (msg['uid'], flags))
- file.flush()
- if self.dofsync:
- os.fsync(file.fileno())
- file.close()
- os.rename(self.filename + ".tmp", self.filename)
-
- if self.dofsync:
- try:
- fd = os.open(os.path.dirname(self.filename), os.O_RDONLY)
- os.fsync(fd)
- os.close(fd)
- except:
- pass
-
- finally:
- self.savelock.release()
+ self.connection.commit()
def getmessagelist(self):
+ if self.isnewfolder():
+ self.messagelist = {}
+ return
+
+ self.messagelist = {}
+ self.cursor.execute('SELECT id,flags from status')
+ for row in self.cursor:
+ flags = [x for x in row[1]]
+ self.messagelist[row[0]] = {'uid': row[0], 'flags': flags}
+
return self.messagelist
+ def uidexists(self,uid):
+ self.cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid})
+ for row in self.cursor:
+ if(row[0]==uid):
+ return 1
+ return 0
+
+ def getmessageuidlist(self):
+ self.cursor.execute('SELECT id from status')
+ r = []
+ for row in self.cursor:
+ r.append(row[0])
+ return r
+
+ def getmessagecount(self):
+ self.cursor.execute('SELECT count(id) from status');
+ row = self.cursor.fetchone()
+ return row[0]
+
def savemessage(self, uid, content, flags, rtime):
if uid < 0:
# We cannot assign a uid.
return uid
- if uid in self.messagelist: # already have it
+ if self.uidexists(uid): # already have it
self.savemessageflags(uid, flags)
return uid
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
+ flags.sort()
+ flags = ''.join(flags)
+ self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)',
+ (uid,flags))
self.autosave()
return uid
def getmessageflags(self, uid):
- return self.messagelist[uid]['flags']
+ self.cursor.execute('SELECT flags FROM status WHERE id=:id',
+ {'id': uid})
+ for row in self.cursor:
+ flags = [x for x in row[0]]
+ return flags
+ return flags
def getmessagetime(self, uid):
return self.messagelist[uid]['time']
def savemessageflags(self, uid, flags):
- self.messagelist[uid]['flags'] = flags
+ self.messagelist[uid] = {'uid': uid, 'flags': flags}
+ flags.sort()
+ flags = ''.join(flags)
+ self.cursor.execute('UPDATE status SET flags=? WHERE id=?',(flags,uid))
self.autosave()
def deletemessage(self, uid):
@@ -151,4 +199,5 @@ class LocalStatusFolder(BaseFolder):
for uid in uidlist:
del(self.messagelist[uid])
- self.autosave()
+ #if self.uidexists(uid):
+ self.cursor.execute('DELETE FROM status WHERE id=:id', {'id': uid})
diff --git a/patch b/patch
new file mode 100644
index 0000000..f089dfe
--- /dev/null
+++ b/patch
@@ -0,0 +1,342 @@
+diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py
+index 5975332..e64965b 100644
+--- a/offlineimap/accounts.py
++++ b/offlineimap/accounts.py
+@@ -316,13 +316,13 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
+ ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
+ ui.loadmessagelist(localrepos, localfolder)
+ localfolder.cachemessagelist()
+- ui.messagelistloaded(localrepos, localfolder, len(localfolder.getmessagelist().keys()))
++ ui.messagelistloaded(localrepos, localfolder, localfolder.getmessagecount())
+
+ # If either the local or the status folder has messages and there is a UID
+ # validity problem, warn and abort. If there are no messages, UW IMAPd
+ # loses UIDVALIDITY. But we don't really need it if both local folders are
+ # empty. So, in that case, just save it off.
+- if len(localfolder.getmessagelist()) or len(statusfolder.getmessagelist()):
++ if localfolder.getmessagecount() or statusfolder.getmessagecount():
+ if not localfolder.isuidvalidityok():
+ ui.validityproblem(localfolder)
+ localrepos.restore_atime()
+@@ -339,7 +339,7 @@ def syncfolder(accountname, remoterepos, remotefolder, localrepos,
+ ui.loadmessagelist(remoterepos, remotefolder)
+ remotefolder.cachemessagelist()
+ ui.messagelistloaded(remoterepos, remotefolder,
+- len(remotefolder.getmessagelist().keys()))
++ remotefolder.getmessagecount())
+
+
+ #
+diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py
+index 0f38bc3..35ab450 100644
+--- a/offlineimap/folder/Base.py
++++ b/offlineimap/folder/Base.py
+@@ -130,6 +130,24 @@ class BaseFolder:
+ You must call cachemessagelist() before calling this function!"""
+ raise NotImplementedException
+
++ def uidexists(self,uid):
++ """Returns true if uid exists"""
++ mlist = self.getmessagelist()
++ if uid in mlist:
++ return 1
++ else:
++ return 0
++ return 0
++
++ def getmessageuidlist(self):
++ """Gets a list of UIDs.
++ You may have to call cachemessagelist() before calling this function!"""
++ return self.getmessagelist().keys()
++
++ def getmessagecount(self):
++ """Gets the number of messages."""
++ return len(self.getmessagelist().keys())
++
+ def getmessage(self, uid):
+ """Returns the content of the specified message."""
+ raise NotImplementedException
+@@ -238,7 +256,7 @@ class BaseFolder:
+ and once that succeeds, get the UID, add it to the others for real,
+ add it to local for real, and delete the fake one."""
+
+- uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
++ uidlist = [uid for uid in self.getmessageuidlist() if uid < 0]
+ threads = []
+
+ usethread = None
+@@ -300,11 +318,10 @@ class BaseFolder:
+ them to dest."""
+ threads = []
+
+- dest_messagelist = dest.getmessagelist()
+- for uid in self.getmessagelist().keys():
++ for uid in self.getmessageuidlist():
+ if uid < 0: # Ignore messages that pass 1 missed.
+ continue
+- if not uid in dest_messagelist:
++ if not dest.uidexists(uid):
+ if self.suggeststhreads():
+ self.waitforthread()
+ thread = InstanceLimitedThread(\
+@@ -327,11 +344,10 @@ class BaseFolder:
+ Look for message present in dest but not in self.
+ If any, delete them."""
+ deletelist = []
+- self_messagelist = self.getmessagelist()
+- for uid in dest.getmessagelist().keys():
++ for uid in dest.getmessageuidlist():
+ if uid < 0:
+ continue
+- if not uid in self_messagelist:
++ if not self.uidexists(uid):
+ deletelist.append(uid)
+ if len(deletelist):
+ UIBase.getglobalui().deletingmessages(deletelist, applyto)
+@@ -354,7 +370,7 @@ class BaseFolder:
+ addflaglist = {}
+ delflaglist = {}
+
+- for uid in self.getmessagelist().keys():
++ for uid in self.getmessageuidlist():
+ if uid < 0: # Ignore messages missed by pass 1
+ continue
+ selfflags = self.getmessageflags(uid)
+diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py
+index c90d0e5..133bdff 100644
+--- a/offlineimap/folder/IMAP.py
++++ b/offlineimap/folder/IMAP.py
+@@ -56,7 +56,7 @@ class IMAPFolder(BaseFolder):
+ return self.accountname
+
+ def suggeststhreads(self):
+- return 1
++ return 0
+
+ def waitforthread(self):
+ self.imapserver.connectionwait()
+diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py
+index 157989d..0972a36 100644
+--- a/offlineimap/folder/LocalStatus.py
++++ b/offlineimap/folder/LocalStatus.py
+@@ -19,9 +19,17 @@
+ from Base import BaseFolder
+ import os, threading
+
++from pysqlite2 import dbapi2 as sqlite
++
+ magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1"
++newmagicline = "OFFLINEIMAP LocalStatus NOW IN SQLITE, DO NOT MODIFY"
+
+ class LocalStatusFolder(BaseFolder):
++ def __deinit__(self):
++ self.save()
++ self.cursor.close()
++ self.connection.close()
++
+ def __init__(self, root, name, repository, accountname, config):
+ self.name = name
+ self.root = root
+@@ -30,12 +38,51 @@ class LocalStatusFolder(BaseFolder):
+ self.dofsync = config.getdefaultboolean("general", "fsync", True)
+ self.filename = os.path.join(root, name)
+ self.filename = repository.getfolderfilename(name)
+- self.messagelist = None
++ self.messagelist = {}
+ self.repository = repository
+ self.savelock = threading.Lock()
+ self.doautosave = 1
+ self.accountname = accountname
+ BaseFolder.__init__(self)
++ self.dbfilename = self.filename + '.sqlite'
++
++ # MIGRATE
++ if os.path.exists(self.filename):
++ self.connection = sqlite.connect(self.dbfilename)
++ self.cursor = self.connection.cursor()
++ self.cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))')
++ if self.isnewfolder():
++ self.messagelist = {}
++ return
++ file = open(self.filename, "rt")
++ self.messagelist = {}
++ line = file.readline().strip()
++ assert(line == magicline)
++ for line in file.xreadlines():
++ line = line.strip()
++ uid, flags = line.split(':')
++ uid = long(uid)
++ flags = [x for x in flags]
++ flags.sort()
++ flags = ''.join(flags)
++ self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)',
++ (uid,flags))
++ file.close()
++ self.connection.commit()
++ os.rename(self.filename, self.filename + ".old")
++ self.cursor.close()
++ self.connection.close()
++
++ # create new
++ if not os.path.exists(self.dbfilename):
++ self.connection = sqlite.connect(self.dbfilename)
++ self.cursor = self.connection.cursor()
++ self.cursor.execute('CREATE TABLE status (id INTEGER PRIMARY KEY, flags VARCHAR(50))')
++ else:
++ self.connection = sqlite.connect(self.dbfilename)
++ self.cursor = self.connection.cursor()
++
++
+
+ def getaccountname(self):
+ return self.accountname
+@@ -44,7 +91,7 @@ class LocalStatusFolder(BaseFolder):
+ return 0
+
+ def isnewfolder(self):
+- return not os.path.exists(self.filename)
++ return not os.path.exists(self.dbfilename)
+
+ def getname(self):
+ return self.name
+@@ -60,84 +107,85 @@ class LocalStatusFolder(BaseFolder):
+
+ def deletemessagelist(self):
+ if not self.isnewfolder():
+- os.unlink(self.filename)
++ self.cursor.close()
++ self.connection.close()
++ os.unlink(self.dbfilename)
+
+ def cachemessagelist(self):
+- if self.isnewfolder():
+- self.messagelist = {}
+- return
+- file = open(self.filename, "rt")
+- self.messagelist = {}
+- line = file.readline().strip()
+- if not line and not line.read():
+- # The status file is empty - should not have happened,
+- # but somehow did.
+- file.close()
+- return
+- assert(line == magicline)
+- for line in file.xreadlines():
+- line = line.strip()
+- uid, flags = line.split(':')
+- uid = long(uid)
+- flags = [x for x in flags]
+- self.messagelist[uid] = {'uid': uid, 'flags': flags}
+- file.close()
++ return
+
+ def autosave(self):
+ if self.doautosave:
+ self.save()
+
+ def save(self):
+- self.savelock.acquire()
+- try:
+- file = open(self.filename + ".tmp", "wt")
+- file.write(magicline + "\n")
+- for msg in self.messagelist.values():
+- flags = msg['flags']
+- flags.sort()
+- flags = ''.join(flags)
+- file.write("%s:%s\n" % (msg['uid'], flags))
+- file.flush()
+- if self.dofsync:
+- os.fsync(file.fileno())
+- file.close()
+- os.rename(self.filename + ".tmp", self.filename)
+-
+- if self.dofsync:
+- try:
+- fd = os.open(os.path.dirname(self.filename), os.O_RDONLY)
+- os.fsync(fd)
+- os.close(fd)
+- except:
+- pass
+-
+- finally:
+- self.savelock.release()
++ self.connection.commit()
+
+ def getmessagelist(self):
++ if self.isnewfolder():
++ self.messagelist = {}
++ return
++
++ self.messagelist = {}
++ self.cursor.execute('SELECT id,flags from status')
++ for row in self.cursor:
++ flags = [x for x in row[1]]
++ self.messagelist[row[0]] = {'uid': row[0], 'flags': flags}
++
+ return self.messagelist
+
++ def uidexists(self,uid):
++ self.cursor.execute('SELECT id FROM status WHERE id=:id',{'id': uid})
++ for row in self.cursor:
++ if(row[0]==uid):
++ return 1
++ return 0
++
++ def getmessageuidlist(self):
++ self.cursor.execute('SELECT id from status')
++ r = []
++ for row in self.cursor:
++ r.append(row[0])
++ return r
++
++ def getmessagecount(self):
++ self.cursor.execute('SELECT count(id) from status');
++ row = self.cursor.fetchone()
++ return row[0]
++
+ def savemessage(self, uid, content, flags, rtime):
+ if uid < 0:
+ # We cannot assign a uid.
+ return uid
+
+- if uid in self.messagelist: # already have it
++ if self.uidexists(uid): # already have it
+ self.savemessageflags(uid, flags)
+ return uid
+
+ self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
++ flags.sort()
++ flags = ''.join(flags)
++ self.cursor.execute('INSERT INTO status (id,flags) VALUES (?,?)',
++ (uid,flags))
+ self.autosave()
+ return uid
+
+ def getmessageflags(self, uid):
+- return self.messagelist[uid]['flags']
++ self.cursor.execute('SELECT flags FROM status WHERE id=:id',
++ {'id': uid})
++ for row in self.cursor:
++ flags = [x for x in row[0]]
++ return flags
++ return flags
+
+ def getmessagetime(self, uid):
+ return self.messagelist[uid]['time']
+
+ def savemessageflags(self, uid, flags):
+- self.messagelist[uid]['flags'] = flags
++ self.messagelist[uid] = {'uid': uid, 'flags': flags}
++ flags.sort()
++ flags = ''.join(flags)
++ self.cursor.execute('UPDATE status SET flags=? WHERE id=?',(flags,uid))
+ self.autosave()
+
+ def deletemessage(self, uid):
+@@ -151,4 +199,5 @@ class LocalStatusFolder(BaseFolder):
+
+ for uid in uidlist:
+ del(self.messagelist[uid])
+- self.autosave()
++ #if self.uidexists(uid):
++ self.cursor.execute('DELETE FROM status WHERE id=:id', {'id': uid})
--
1.7.1
More information about the OfflineIMAP-project
mailing list