[PATCH v2] fix: when called with -a, mbnames must not erase entries of other accounts
Nicolas Sebrecht
nicolas.s-dev at laposte.net
Thu Jun 23 12:48:13 BST 2016
Make mbnames to work with intermediate files, one per account, in the JSON
format. The mbnames target is built from those intermediate files.
Github-Fix: https://github.com/OfflineIMAP/offlineimap/issues/66
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev at laposte.net>
---
-- interdiff --
diff --git a/README.md b/README.md
index 4074e5f..544f5be 100644
--- a/README.md
+++ b/README.md
@@ -83,7 +83,6 @@ Bugs, issues and contributions can be requested to both the mailing list or the
* Python v2.7+
* Python v3.4+ ***(experimental)***
* six (required)
-* yaml (required if mbnames is enabled)
* imaplib2 >= 2.55 (optional)
diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py
index 538bf9e..400439e 100644
--- a/offlineimap/mbnames.py
+++ b/offlineimap/mbnames.py
@@ -15,7 +15,9 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
import re # For folderfilter.
+import json
from threading import Lock
from os import listdir, makedirs, path
from sys import exc_info
@@ -26,7 +28,6 @@ except ImportError:
from collections import UserDict
-_yaml = None # Make non-standard yaml import optional.
_mbLock = Lock()
_mbnames = None
@@ -39,11 +40,9 @@ def add(accountname, folder_root, foldername):
_mbnames.addAccountFolder(accountname, folder_root, foldername)
def init(conf, ui, dry_run):
- global _mbnames, _yaml
+ global _mbnames
enabled = conf.getdefaultboolean("mbnames", "enabled", False)
if enabled is True and _mbnames is None:
- import yaml
- _yaml = yaml
_mbnames = _Mbnames(conf, ui, dry_run)
def write():
@@ -90,7 +89,7 @@ class _IntermediateMbnames(object):
return self._folder_root
def write(self):
- """Write intermediate mbnames file in yaml format."""
+ """Write intermediate mbnames file in JSON format."""
itemlist = []
@@ -104,7 +103,7 @@ class _IntermediateMbnames(object):
if not self._dryrun:
with open(self._path, "wt") as intermediateFile:
- intermediateFile.write(_yaml.dump(itemlist))
+ json.dump(itemlist, intermediateFile)
class _Mbnames(object):
@@ -175,10 +174,10 @@ class _Mbnames(object):
try:
for foo in listdir(self._mbnamesdir):
foo = path.join(self._mbnamesdir, foo)
- if path.isfile(foo) and foo[-4:] == '.yml':
+ if path.isfile(foo) and foo[-5:] == '.json':
try:
with open(foo, 'rt') as intermediateFile:
- for item in _yaml.load(intermediateFile):
+ for item in json.load(intermediateFile):
itemlist.append(item)
except Exception as e:
self.ui.error(
-- /interdiff --
The following changes since commit 52120beb27a718dbec3d88a00b3448fcb5137777:
man: offlineimapui: minor typo fix (2016-06-19 23:32:01 +0200)
are available in the git repository at:
https://github.com/nicolas33/offlineimap.git ns/mbnames-2
for you to fetch changes up to edb8ba9fa8c548b7221385a7330a48dbcc5c18fd:
fix: when called with -a, mbnames must not erase entries of other accounts (2016-06-23 13:43:12 +0200)
----------------------------------------------------------------
offlineimap/accounts.py | 10 +-
offlineimap/globals.py | 4 +-
offlineimap/init.py | 10 +-
offlineimap/mbnames.py | 256 ++++++++++++++++++++++++++++++++++-------------
offlineimap/ui/UIBase.py | 4 +-
5 files changed, 200 insertions(+), 84 deletions(-)
diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py
index 734078f..36a69ab 100644
--- a/offlineimap/accounts.py
+++ b/offlineimap/accounts.py
@@ -369,9 +369,7 @@ class SyncableAccount(Account):
# wait for all threads to finish
for thr in folderthreads:
thr.join()
- # Write out mailbox names if required and not in dry-run mode
- if not self.dryrun:
- mbnames.write(False)
+ mbnames.writeIntermediateFile(self.name) # Write out mailbox names.
localrepos.forgetfolders()
remoterepos.forgetfolders()
except:
@@ -512,9 +510,9 @@ def syncfolder(account, remotefolder, quick):
# Load local folder.
localfolder = account.get_local_folder(remotefolder)
- # Write the mailboxes
- mbnames.add(account.name, localfolder.getname(),
- localrepos.getlocalroot())
+ # Add the folder to the mbnames mailboxes.
+ mbnames.add(account.name, localrepos.getlocalroot(),
+ localfolder.getname())
# Load status folder.
statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().
diff --git a/offlineimap/globals.py b/offlineimap/globals.py
index 3415718..e049988 100644
--- a/offlineimap/globals.py
+++ b/offlineimap/globals.py
@@ -8,6 +8,6 @@ from offlineimap.utils import const
options = const.ConstProxy()
def set_options(source):
- """ Sets the source for options variable """
+ """Sets the source for options variable."""
- options.set_source (source)
+ options.set_source(source)
diff --git a/offlineimap/init.py b/offlineimap/init.py
index 9815eca..37df977 100644
--- a/offlineimap/init.py
+++ b/offlineimap/init.py
@@ -25,8 +25,7 @@ import logging
from optparse import OptionParser
import offlineimap
-from offlineimap import accounts, threadutil, folder
-from offlineimap import globals
+from offlineimap import globals, accounts, threadutil, folder, mbnames
from offlineimap.ui import UI_LIST, setglobalui, getglobalui
from offlineimap.CustomConfig import CustomConfigParser
from offlineimap.utils import stacktrace
@@ -423,7 +422,7 @@ class OfflineImap(object):
signal.signal(signal.SIGQUIT, sig_handler)
# Various initializations that need to be performed:
- offlineimap.mbnames.init(self.config, syncaccounts)
+ mbnames.init(self.config, self.ui, options.dryrun)
if options.singlethreading:
# Singlethreaded.
@@ -440,9 +439,8 @@ class OfflineImap(object):
t.start()
threadutil.monitor()
- if not options.dryrun:
- offlineimap.mbnames.write(True)
-
+ # All sync are done.
+ mbnames.write()
self.ui.terminate()
return 0
except (SystemExit):
diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py
index 8829ee5..400439e 100644
--- a/offlineimap/mbnames.py
+++ b/offlineimap/mbnames.py
@@ -1,6 +1,5 @@
# Mailbox name generator
-#
-# Copyright (C) 2002-2015 John Goerzen & contributors
+# Copyright (C) 2002-2016 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -16,80 +15,201 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-import os.path
-import re # for folderfilter
+
+import re # For folderfilter.
+import json
from threading import Lock
+from os import listdir, makedirs, path
+from sys import exc_info
+try:
+ import UserDict
+except ImportError:
+ # Py3
+ from collections import UserDict
+
-boxes = {}
-localroots = {}
-config = None
-accounts = None
-mblock = Lock()
-
-def init(conf, accts):
- global config, accounts
- config = conf
- accounts = accts
-
-def add(accountname, foldername, localfolders):
- if not accountname in boxes:
- boxes[accountname] = []
- localroots[accountname] = localfolders
- if not foldername in boxes[accountname]:
- boxes[accountname].append(foldername)
-
-def write(allcomplete):
- incremental = config.getdefaultboolean("mbnames", "incremental", False)
-
- # Skip writing if we don't want incremental writing and we're not done.
- if not incremental and not allcomplete:
+_mbLock = Lock()
+_mbnames = None
+
+
+def add(accountname, folder_root, foldername):
+ if _mbnames is None:
return
- # Skip writing if we want incremental writing and we're done.
- if incremental and allcomplete:
+ with _mbLock:
+ _mbnames.addAccountFolder(accountname, folder_root, foldername)
+
+def init(conf, ui, dry_run):
+ global _mbnames
+ enabled = conf.getdefaultboolean("mbnames", "enabled", False)
+ if enabled is True and _mbnames is None:
+ _mbnames = _Mbnames(conf, ui, dry_run)
+
+def write():
+ """Write the mbnames file."""
+
+ global _mbnames
+
+ if _mbnames is None:
return
- # See if we're ready to write it out.
- for account in accounts:
- if account not in boxes:
- return
+ if _mbnames.get_incremental() is not True:
+ _mbnames.write()
- __genmbnames()
+def writeIntermediateFile(accountname):
+ """Write intermediate mbnames file."""
-def __genmbnames():
- """Takes a configparser object and a boxlist, which is a list of hashes
- containing 'accountname' and 'foldername' keys."""
+ global _mbnames
+
+ if _mbnames is None:
+ return
+
+ _mbnames.writeIntermediateFile(accountname)
+ if _mbnames.get_incremental() is True:
+ _mbnames.write()
+
+
+class _IntermediateMbnames(object):
+ """mbnames data for one account."""
+
+ def __init__(self, accountname, folder_root, mbnamesdir, folderfilter,
+ dry_run):
+
+ self._foldernames = []
+ self._accountname = accountname
+ self._folder_root = folder_root
+ self._folderfilter = folderfilter
+ self._path = path.join(mbnamesdir, "%s.yml"% accountname)
+ self._dryrun = dry_run
+
+ def add(self, foldername):
+ self._foldernames.append(foldername)
+
+ def get_folder_root(self):
+ return self._folder_root
+
+ def write(self):
+ """Write intermediate mbnames file in JSON format."""
+
+ itemlist = []
+
+ for foldername in self._foldernames:
+ if self._folderfilter(self._accountname, foldername):
+ itemlist.append({
+ 'accountname': self._accountname,
+ 'foldername': foldername,
+ 'localfolders': self._folder_root,
+ })
+
+ if not self._dryrun:
+ with open(self._path, "wt") as intermediateFile:
+ json.dump(itemlist, intermediateFile)
+
+
+class _Mbnames(object):
+ def __init__(self, config, ui, dry_run):
+
+ self._config = config
+ self._dryrun = dry_run
+ self.ui = ui
+
+ # Keys: accountname, values: _IntermediateMbnames instance
+ self._intermediates = {}
+ self._incremental = None
+ self._mbnamesdir = None
+ self._path = None
+ self._folderfilter = lambda accountname, foldername: True
+ self._func_sortkey = lambda d: (d['accountname'], d['foldername'])
+ self._peritem = self._config.get("mbnames", "peritem", raw=1)
- xforms = [os.path.expanduser, os.path.expandvars]
- mblock.acquire()
- try:
localeval = config.getlocaleval()
- if not config.getdefaultboolean("mbnames", "enabled", 0):
- return
- path = config.apply_xforms(config.get("mbnames", "filename"), xforms)
- file = open(path, "wt")
- file.write(localeval.eval(config.get("mbnames", "header")))
- folderfilter = lambda accountname, foldername: 1
- if config.has_option("mbnames", "folderfilter"):
- folderfilter = localeval.eval(config.get("mbnames", "folderfilter"),
- {'re': re})
- mb_sort_keyfunc = lambda d: (d['accountname'], d['foldername'])
- if config.has_option("mbnames", "sort_keyfunc"):
- mb_sort_keyfunc = localeval.eval(config.get("mbnames", "sort_keyfunc"),
- {'re': re})
+ self._header = localeval.eval(config.get("mbnames", "header"))
+ self._sep = localeval.eval(config.get("mbnames", "sep"))
+ self._footer = localeval.eval(config.get("mbnames", "footer"))
+
+ mbnamesdir = path.join(config.getmetadatadir(), "mbnames")
+ try:
+ if not self._dryrun:
+ makedirs(mbnamesdir)
+ except OSError:
+ pass
+ self._mbnamesdir = mbnamesdir
+
+ xforms = [path.expanduser, path.expandvars]
+ self._path = config.apply_xforms(
+ config.get("mbnames", "filename"), xforms)
+
+ if self._config.has_option("mbnames", "sort_keyfunc"):
+ self._func_sortkey = localeval.eval(
+ self._config.get("mbnames", "sort_keyfunc"), {'re': re})
+
+ if self._config.has_option("mbnames", "folderfilter"):
+ self._folderfilter = localeval.eval(
+ self._config.get("mbnames", "folderfilter"), {'re': re})
+
+ def addAccountFolder(self, accountname, folder_root, foldername):
+ """Add foldername entry for an account."""
+
+ if accountname not in self._intermediates:
+ self._intermediates[accountname] = _IntermediateMbnames(
+ accountname,
+ folder_root,
+ self._mbnamesdir,
+ self._folderfilter,
+ self._dryrun,
+ )
+
+ self._intermediates[accountname].add(foldername)
+
+ def get_incremental(self):
+ if self._incremental is None:
+ self._incremental = self._config.getdefaultboolean(
+ "mbnames", "incremental", False)
+
+ return self._incremental
+
+ def write(self):
itemlist = []
- for accountname in boxes.keys():
- localroot = localroots[accountname]
- for foldername in boxes[accountname]:
- if folderfilter(accountname, foldername):
- itemlist.append({'accountname': accountname,
- 'foldername': foldername,
- 'localfolders': localroot})
- itemlist.sort(key = mb_sort_keyfunc)
- format_string = config.get("mbnames", "peritem", raw=1)
- itemlist = [format_string % d for d in itemlist]
- file.write(localeval.eval(config.get("mbnames", "sep")).join(itemlist))
- file.write(localeval.eval(config.get("mbnames", "footer")))
- file.close()
- finally:
- mblock.release()
+
+ try:
+ for foo in listdir(self._mbnamesdir):
+ foo = path.join(self._mbnamesdir, foo)
+ if path.isfile(foo) and foo[-5:] == '.json':
+ try:
+ with open(foo, 'rt') as intermediateFile:
+ for item in json.load(intermediateFile):
+ itemlist.append(item)
+ except Exception as e:
+ self.ui.error(
+ e,
+ exc_info()[2],
+ "intermediate mbnames file %s not properly read"% foo
+ )
+ except OSError:
+ pass
+
+ itemlist.sort(key=self._func_sortkey)
+ itemlist = [self._peritem % d for d in itemlist]
+
+ if not self._dryrun:
+ try:
+ with open(self._path, 'wt') as mbnamesFile:
+ mbnamesFile.write(self._header)
+ mbnamesFile.write(self._sep.join(itemlist))
+ mbnamesFile.write(self._footer)
+ except (OSError, IOError) as e:
+ self.ui.error(
+ e,
+ exc_info()[2],
+ "mbnames file %s not properly written"% self._path
+ )
+
+ def writeIntermediateFile(self, accountname):
+ try:
+ self._intermediates[accountname].write()
+ except (OSError, IOError) as e:
+ self.ui.error(
+ e,
+ exc_info()[2],
+ "intermediate mbnames file %s not properly written"% self._path
+ )
diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py
index 62e42d1..ef9bb37 100644
--- a/offlineimap/ui/UIBase.py
+++ b/offlineimap/ui/UIBase.py
@@ -1,5 +1,5 @@
# UI base class
-# Copyright (C) 2002-2015 John Goerzen & contributors
+# Copyright (C) 2002-2016 John Goerzen & contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -148,7 +148,7 @@ class UIBase(object):
of the sync run when offlineiamp exits. It is recommended to
always pass in exceptions if possible, so we can give the user
the best debugging info.
-
+
We are always pushing tracebacks to the exception queue to
make them to be output at the end of the run to allow users
pass sensible diagnostics to the developers or to solve
--
2.7.4
More information about the OfflineIMAP-project
mailing list