[PATCH 4/4] learn unicode support
Nicolas Sebrecht
nicolas.s-dev at laposte.net
Tue Feb 10 17:04:43 GMT 2015
This feature is NOT WORKING. This is EXPERIMENTAL WIP.
Also, such change is very intrusive/invasive by nature. While the full ASCII
mode is still the default, it is impossible to not impact legacy code when run
without unicode support.
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev at laposte.net>
---
offlineimap.conf | 8 +-
offlineimap/CustomConfig.py | 30 +++--
offlineimap/__init__.py | 2 +-
offlineimap/accounts.py | 68 +++++++---
offlineimap/folder/Base.py | 78 ++++++-----
offlineimap/folder/Gmail.py | 12 +-
offlineimap/folder/GmailMaildir.py | 8 +-
offlineimap/folder/IMAP.py | 137 ++++++++++++-------
offlineimap/folder/LocalStatus.py | 47 ++++++-
offlineimap/folder/LocalStatusSQLite.py | 15 ++-
offlineimap/folder/Maildir.py | 55 ++++++--
offlineimap/folder/UIDMaps.py | 27 +++-
offlineimap/imaplibutil.py | 35 +++--
offlineimap/imapserver.py | 84 ++++++------
offlineimap/imaputil.py | 19 +--
offlineimap/init.py | 89 +++++++++----
offlineimap/localeval.py | 12 +-
offlineimap/mbnames.py | 38 ++++--
offlineimap/repository/Base.py | 24 ++--
offlineimap/repository/IMAP.py | 42 ++++--
offlineimap/repository/LocalStatus.py | 22 +++-
offlineimap/repository/Maildir.py | 102 +++++++++++----
offlineimap/repository/__init__.py | 9 +-
offlineimap/threadutil.py | 6 +-
offlineimap/ui/Curses.py | 24 +++-
offlineimap/ui/Machine.py | 27 +++-
offlineimap/ui/TTY.py | 41 ++++--
offlineimap/ui/UIBase.py | 225 +++++++++++++++++++-------------
offlineimap/ui/debuglock.py | 23 +++-
offlineimap/utils/const.py | 5 +-
30 files changed, 900 insertions(+), 414 deletions(-)
diff --git a/offlineimap.conf b/offlineimap.conf
index cfafb23..0ad3bcc 100644
--- a/offlineimap.conf
+++ b/offlineimap.conf
@@ -5,8 +5,14 @@
# More details can be found in the included user documention, which is
# also available at: http://docs.offlineimap.org/en/latest/
+# NOTE 0: This file is read with UTF-8 encoding when option is enabled from
+# command line. This means iso-8859-1 and ASCII charsets are natively
+# supported. Otherwise, this file is expected to be full ASCII.
+#
+# If any other charset is used somewhere with success, it's just because you're
+# lucky.
-# NOTE 1: Settings generally support python interpolation. This means
+# NOTE1: Settings generally support python interpolation. This means
# values can contain python format strings which refer to other values
# in the same section, or values in a special DEFAULT section. This
# allows you for example to use common settings for multiple accounts:
diff --git a/offlineimap/CustomConfig.py b/offlineimap/CustomConfig.py
index 44cfcab..8820a78 100644
--- a/offlineimap/CustomConfig.py
+++ b/offlineimap/CustomConfig.py
@@ -22,7 +22,10 @@ try:
from ConfigParser import SafeConfigParser, Error
except ImportError: #python3
from configparser import SafeConfigParser, Error
+
from offlineimap.localeval import LocalEval
+from offlineimap.utils import uni
+from offlineimap import globals
class CustomConfigParser(SafeConfigParser):
def __init__(self):
@@ -75,7 +78,8 @@ class CustomConfigParser(SafeConfigParser):
val = self.get(section, option).strip()
return re.split(separator_re, val)
except re.error as e:
- raise Error("Bad split regexp '%s': %s" % \
+ separator_re = uni.uni2fs(separator_re)
+ raise Error("Bad split regexp '%s': %s"%
(separator_re, e)), None, exc_info()[2]
def getdefaultlist(self, section, option, default, separator_re):
@@ -88,9 +92,20 @@ class CustomConfigParser(SafeConfigParser):
return default
def getmetadatadir(self):
+ """Returned metadatadir is encoded string of bytes.
+
+ This is an exception to the encode as late as possible philosophy
+ because of the way we use it. It's mostly used as a prefix of the path
+ which we append str values in full ASCII. Also, it's used in a lot of
+ places in the code. Not encoding here would require to encode the path
+ everywhere. So, it looks simpler to encode it here."""
+
xforms = [os.path.expanduser, os.path.expandvars]
d = self.getdefault("general", "metadata", "~/.offlineimap")
metadatadir = self.apply_xforms(d, xforms)
+ if globals.options.use_unicode:
+ metadatadir = uni.uni2fs(metadatadir, exception_msg=
+ "cannot convert encoding for metadata path %s"% uni.uni2std(metadatadir))
if not os.path.exists(metadatadir):
os.mkdir(metadatadir, 0o700)
return metadatadir
@@ -102,8 +117,7 @@ class CustomConfigParser(SafeConfigParser):
xforms = [os.path.expanduser, os.path.expandvars]
if self.has_option("general", "pythonfile"):
- path = self.get("general", "pythonfile")
- path = self.apply_xforms(path, xforms)
+ path = self.apply_xforms(self.get("general", "pythonfile"), xforms)
else:
path = None
@@ -132,22 +146,22 @@ class CustomConfigParser(SafeConfigParser):
self.set(section, option, value)
- def apply_xforms(self, string, transforms):
+ def apply_xforms(self, s, transforms):
"""Applies set of transformations to a string.
Arguments:
- - string: source string; if None, then no processing will
+ - s: source string; if None, then no processing will
take place.
- transforms: iterable that returns transformation function
on each turn.
Returns transformed string."""
- if string == None:
+ if s == None:
return None
for f in transforms:
- string = f(string)
- return string
+ s = f(s)
+ return s
diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py
index 441af44..1126781 100644
--- a/offlineimap/__init__.py
+++ b/offlineimap/__init__.py
@@ -9,7 +9,7 @@ __author__ = "John Goerzen"
__author_email__= "john at complete.org"
__description__ = "Disconnected Universal IMAP Mail Synchronization/Reader Support"
__license__ = "Licensed under the GNU GPL v2 or any later version (with an OpenSSL exception)"
-__bigcopyright__ = """%(__productname__)s %(__bigversion__)s
+__bigcopyright__ = u"""%(__productname__)s %(__bigversion__)s
%(__license__)s""" % locals()
__homepage__ = "http://offlineimap.org"
diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py
index 62ed5c3..eb6f97a 100644
--- a/offlineimap/accounts.py
+++ b/offlineimap/accounts.py
@@ -25,6 +25,7 @@ from offlineimap import globals
from offlineimap.repository import Repository
from offlineimap.ui import getglobalui
from offlineimap.threadutil import InstanceLimitedThread
+from offlineimap.utils import uni
try:
import fcntl
@@ -99,6 +100,9 @@ class Account(CustomConfig.ConfigHelperMixin):
return self.name
def __str__(self):
+ # accounts names are expected full ASCII but just in case...
+ if globals.options.use_unicode:
+ return uni.uni2fs(self.name)
return self.name
def getaccountmeta(self):
@@ -336,20 +340,37 @@ class SyncableAccount(Account):
if Account.abort_NOW_signal.is_set(): break
if not remotefolder.sync_this:
- self.ui.debug('', "Not syncing filtered folder '%s'"
- "[%s]"% (remotefolder, remoterepos))
+ self.ui.debug('', u"Not syncing filtered folder '%s'[%s]"%
+ (remotefolder, remoterepos))
continue # Ignore filtered folder
localfolder = self.get_local_folder(remotefolder)
if not localfolder.sync_this:
- self.ui.debug('', "Not syncing filtered folder '%s'"
- "[%s]"% (localfolder, localfolder.repository))
+ self.ui.debug('', u"Not syncing filtered folder '%s'[%s]"%
+ (localfolder, localfolder.repository))
continue # Ignore filtered folder
if not globals.options.singlethreading:
- thread = InstanceLimitedThread(\
- instancename = 'FOLDER_' + self.remoterepos.getname(),
- target = syncfolder,
- name = "Folder %s [acc: %s]"% (remotefolder.getexplainedname(), self),
- args = (self, remotefolder, quick))
+ reponame = self.remoterepos.getname()
+ remotefoldername = remotefolder.getexplainedname()
+ # UNICODE: we have to encode Unicode because Thread module
+ # has str type assumptions.
+ #
+ # It would not much be a problem if these strings were
+ # not used later for logging. As a consequence, encoding
+ # these variable here is wrong in the sense that we encode
+ # far too much early. It requires us to decode them as soon
+ # as we extract the strings from the Thread instances...
+ #
+ # To make it right, parameters for logging and those
+ # dedicated to the Thread class must be uncoupled from each
+ # other.
+ if globals.options.use_unicode:
+ reponame = uni.uni2bytes(reponame)
+ remotefoldername = uni.uni2bytes(remotefoldername)
+ thread = InstanceLimitedThread(
+ instancename= 'FOLDER_%s'% reponame,
+ target= syncfolder,
+ name= "Folder %s [acc: %s]"% (remotefoldername, self),
+ args= (self, remotefolder, quick))
thread.start()
folderthreads.append(thread)
else:
@@ -383,15 +404,21 @@ class SyncableAccount(Account):
if not cmd:
return
try:
- self.ui.callhook("Calling hook: " + cmd)
+ self.ui.callhook(u"Calling hook: "+ cmd)
+ if globals.options.use_unicode:
+ fs_cmd = uni.uni2fs(cmd)
+ else:
+ fs_cmd = cmd
if self.dryrun: # don't if we are in dry-run mode
return
- p = Popen(cmd, shell=True,
+ p = Popen(fs_cmd, shell=True,
stdin=PIPE, stdout=PIPE, stderr=PIPE,
close_fds=True)
r = p.communicate()
- self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n"% r)
- self.ui.callhook("Hook return code: %d"% p.returncode)
+ if globals.options.use_unicode:
+ r = uni.fs2uni(r)
+ self.ui.callhook(u"Hook stdout: %s\nHook stderr:%s\n"% r)
+ self.ui.callhook(u"Hook return code: %d"% p.returncode)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
@@ -472,16 +499,16 @@ def syncfolder(account, remotefolder, quick):
ui.syncingmessages(remoterepos, remotefolder, localrepos, localfolder)
remotefolder.syncmessagesto(localfolder, statusfolder)
else:
- ui.debug('imap', "Not syncing to read-only repository '%s'" \
- % localrepos.getname())
+ ui.debug('imap', u"Not syncing to read-only repository '%s'"%
+ localrepos.getname())
# Synchronize local changes
if not remoterepos.getconfboolean('readonly', False):
ui.syncingmessages(localrepos, localfolder, remoterepos, remotefolder)
localfolder.syncmessagesto(remotefolder, statusfolder)
else:
- ui.debug('', "Not syncing to read-only repository '%s'" \
- % remoterepos.getname())
+ ui.debug('', u"Not syncing to read-only repository '%s'"%
+ remoterepos.getname())
statusfolder.save()
localrepos.restore_atime()
@@ -492,10 +519,11 @@ def syncfolder(account, remotefolder, quick):
if e.severity > OfflineImapError.ERROR.FOLDER:
raise
else:
- ui.error(e, exc_info()[2], msg = "Aborting sync, folder '%s' "
- "[acc: '%s']" % (localfolder, account))
+ msg = u"Aborting sync, folder '%s' [acc: '%s']"% \
+ (localfolder, account)
+ ui.error(e, exc_info()[2], msg)
except Exception as e:
- ui.error(e, msg = "ERROR in syncfolder for %s folder %s: %s"%
+ ui.error(e, msg=u"ERROR in syncfolder for %s folder %s: %s"%
(account, remotefolder.getvisiblename(), traceback.format_exc()))
finally:
for folder in ["statusfolder", "localfolder", "remotefolder"]:
diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py
index 3a04ef6..d0cc66f 100644
--- a/offlineimap/folder/Base.py
+++ b/offlineimap/folder/Base.py
@@ -23,6 +23,7 @@ from offlineimap import threadutil, emailutil
from offlineimap import globals
from offlineimap.ui import getglobalui
from offlineimap.error import OfflineImapError
+from offlineimap.utils import uni
import offlineimap.accounts
@@ -60,10 +61,10 @@ class BaseFolder(object):
repo, "dynamic_folderfilter", False)
self._sync_this = repository.should_sync_folder(self.ffilter_name)
if self._dynamic_folderfilter:
- self.ui.debug('', "Running dynamic folder filtering on '%s'[%s]"%
+ self.ui.debug('', u"Running dynamic folder filtering on '%s'[%s]"%
(self.ffilter_name, repository))
elif not self._sync_this:
- self.ui.debug('', "Filtering out '%s'[%s] due to folderfilter"%
+ self.ui.debug('', u"Filtering out '%s'[%s] due to folderfilter"%
(self.ffilter_name, repository))
# Passes for syncmessagesto
@@ -77,6 +78,9 @@ class BaseFolder(object):
def __str__(self):
# FIMXE: remove calls of this. We have getname().
+ # If still in use, encode folder name.
+ if globals.options.use_unicode:
+ return uni.uni2fs(self.name)
return self.name
@property
@@ -199,8 +203,16 @@ class BaseFolder(object):
def _getuidfilename(self):
"""provides UIDVALIDITY cache filename for class internal purposes.
- return os.path.join(self.repository.getuiddir(),
- self.getfolderbasename())
+ It is str type encoded to filesystem."""
+
+ uiddir = self.repository.getuiddir()
+ folderbasename = self.getfolderbasename()
+ if globals.options.use_unicode:
+ uiddir = uni.uni2fs(uiddir)
+ # Fix the filename to IMAP UTF-7 encoding. This prevent from playing
+ # with multiple cache files representative of the same folder.
+ folderbasename = uni.uni2fs(uni.uni2imap(folderbasename))
+ return os.path.join(uiddir, folderbasename)
def get_saveduidvalidity(self):
"""Return the previously cached UIDVALIDITY value
@@ -493,20 +505,22 @@ class BaseFolder(object):
next line\n
"""
- self.ui.debug('', 'addmessageheader: called to add %s: %s'%
+ self.ui.debug('', u'addmessageheader: called to add %s: %s'%
(headername, headervalue))
insertionpoint = content.find(linebreak * 2)
if insertionpoint == -1:
- self.ui.debug('', 'addmessageheader: headers were missing')
+ self.ui.debug('', u'addmessageheader: headers were missing')
else:
- self.ui.debug('', 'addmessageheader: headers end at position %d' % insertionpoint)
+ self.ui.debug('', u'addmessageheader: headers end at position %d'%
+ insertionpoint)
mark = '==>EOH<=='
contextstart = max(0, insertionpoint - 100)
contextend = min(len(content), insertionpoint + 100)
- self.ui.debug('', 'addmessageheader: header/body transition context (marked by %s): %s' %
- (mark, repr(content[contextstart:insertionpoint]) + \
- mark + repr(content[insertionpoint:contextend])))
+ self.ui.debug('', u'addmessageheader: header/body transition '
+ 'context (marked by %s): %s%s%s'% (mark,
+ repr(content[contextstart:insertionpoint]), mark,
+ repr(content[insertionpoint:contextend])))
# Hoping for case #4
prefix = linebreak
@@ -528,11 +542,11 @@ class BaseFolder(object):
if content[0:len(linebreak)] != linebreak:
suffix = suffix + linebreak
- self.ui.debug('', 'addmessageheader: insertionpoint = %d'% insertionpoint)
+ self.ui.debug('', u'addmessageheader: insertionpoint = %d'% insertionpoint)
headers = content[0:insertionpoint]
- self.ui.debug('', 'addmessageheader: headers = %s'% repr(headers))
+ self.ui.debug('', u'addmessageheader: headers = %s'% repr(headers))
new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix
- self.ui.debug('', 'addmessageheader: new_header = ' + repr(new_header))
+ self.ui.debug('', u'addmessageheader: new_header = %s'% repr(new_header))
return headers + new_header + content[insertionpoint:]
@@ -563,13 +577,13 @@ class BaseFolder(object):
Returns: header value or None if no such header was found
"""
- self.ui.debug('', 'getmessageheader: called to get %s'% name)
+ self.ui.debug('', u'getmessageheader: called to get %s'% name)
eoh = self.__find_eoh(content)
- self.ui.debug('', 'getmessageheader: eoh = %d'% eoh)
+ self.ui.debug('', u'getmessageheader: eoh = %d'% eoh)
headers = content[0:eoh]
- self.ui.debug('', 'getmessageheader: headers = %s'% repr(headers))
+ self.ui.debug('', u'getmessageheader: headers = %s'% repr(headers))
- m = re.search('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE)
+ m = re.search(u'^%s:(.*)$'% name, headers, flags = re.MULTILINE | re.IGNORECASE)
if m:
return m.group(1).strip()
else:
@@ -587,13 +601,13 @@ class BaseFolder(object):
Returns: list of header values or emptylist if no such header was found
"""
- self.ui.debug('', 'getmessageheaderlist: called to get %s' % name)
+ self.ui.debug('', u'getmessageheaderlist: called to get %s'% name)
eoh = self.__find_eoh(content)
- self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh)
+ self.ui.debug('', u'getmessageheaderlist: eoh = %d'% eoh)
headers = content[0:eoh]
- self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers))
+ self.ui.debug('', u'getmessageheaderlist: headers = %s'% repr(headers))
- return re.findall('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE)
+ return re.findall(u'^%s:(.*)$'% name, headers, flags = re.MULTILINE | re.IGNORECASE)
def deletemessageheaders(self, content, header_list):
@@ -608,15 +622,15 @@ class BaseFolder(object):
if type(header_list) != type([]):
header_list = [header_list]
- self.ui.debug('', 'deletemessageheaders: called to delete %s'% (header_list))
+ self.ui.debug('', u'deletemessageheaders: called to delete %s'% (header_list))
if not len(header_list): return content
eoh = self.__find_eoh(content)
- self.ui.debug('', 'deletemessageheaders: end of headers = %d'% eoh)
+ self.ui.debug('', u'deletemessageheaders: end of headers = %d'% eoh)
headers = content[0:eoh]
rest = content[eoh:]
- self.ui.debug('', 'deletemessageheaders: headers = %s'% repr(headers))
+ self.ui.debug('', u'deletemessageheaders: headers = %s'% repr(headers))
new_headers = []
for h in headers.split('\n'):
keep_it = True
@@ -718,8 +732,9 @@ class BaseFolder(object):
self.deletemessage(uid)
else:
raise OfflineImapError("Trying to save msg (uid %d) on folder "
- "%s returned invalid uid %d"% (uid, dstfolder.getvisiblename(),
- new_uid), OfflineImapError.ERROR.MESSAGE)
+ "%s returned invalid uid %d"% (uni.uni2fs(uid),
+ uni.uni2fs(dstfolder.getvisiblename()),
+ uni.uni2fs(new_uid)), OfflineImapError.ERROR.MESSAGE)
except (KeyboardInterrupt): # bubble up CTRL-C
raise
except OfflineImapError as e:
@@ -728,7 +743,7 @@ class BaseFolder(object):
self.ui.error(e, exc_info()[2])
except Exception as e:
self.ui.error(e, exc_info()[2],
- msg = "Copying message %s [acc: %s]"% (uid, self.accountname))
+ msg = u"Copying message %s [acc: %s]"% (uid, self.accountname))
raise #raise on unknown errors, so we can fix those
def __syncmessagesto_copy(self, dstfolder, statusfolder):
@@ -761,10 +776,13 @@ class BaseFolder(object):
# exceptions are caught in copymessageto()
if self.suggeststhreads() and not globals.options.singlethreading:
self.waitforthread()
+ name = u"Copy message from %s:%s"% (self.repository, self)
+ if globals.options.use_unicode:
+ name = uni.uni2bytes(name)
thread = threadutil.InstanceLimitedThread(\
self.getcopyinstancelimit(),
target = self.copymessageto,
- name = "Copy message from %s:%s" % (self.repository, self),
+ name = name,
args = (uid, dstfolder, statusfolder))
thread.start()
threads.append(thread)
@@ -899,8 +917,8 @@ class BaseFolder(object):
raise
self.ui.error(e, exc_info()[2])
except Exception as e:
- self.ui.error(e, exc_info()[2], "Syncing folder %s [acc: %s]" %\
- (self, self.accountname))
+ self.ui.error(e, exc_info()[2], u"Syncing folder %s [acc: %s]"%
+ (self, self.accountname))
raise # raise unknown Exceptions so we can fix them
def __eq__(self, other):
diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py
index 1afbe47..8ca2c5e 100644
--- a/offlineimap/folder/Gmail.py
+++ b/offlineimap/folder/Gmail.py
@@ -21,6 +21,7 @@ from sys import exc_info
from offlineimap import imaputil, OfflineImapError
from offlineimap import imaplibutil
+from offlineimap.utils import uni
import offlineimap.accounts
from .IMAP import IMAPFolder
@@ -141,9 +142,9 @@ class GmailFolder(IMAPFolder):
res_type, response = imapobj.fetch("'%s'"% msgsToFetch,
'(FLAGS X-GM-LABELS UID)')
if res_type != 'OK':
- raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \
- (self.getrepository(), self) + \
- "Server responded '[%s] %s'" % \
+ raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. "%
+ (uni.uni2fs(self.getrepository()), self) +
+ "Server responded '[%s] %s'"%
(res_type, response), OfflineImapError.ERROR.FOLDER), \
None, exc_info()[2]
finally:
@@ -157,9 +158,8 @@ class GmailFolder(IMAPFolder):
messagestr = messagestr.split(' ', 1)[1]
options = imaputil.flags2hash(messagestr)
if not 'UID' in options:
- self.ui.warn('No UID in message with options %s' %\
- str(options),
- minor = 1)
+ self.ui.warn(u'No UID in message with options %s'%
+ str(options), minor=1)
else:
uid = long(options['UID'])
self.messagelist[uid] = self.msglist_item_initializer(uid)
diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py
index 5ca0e1f..d47b40c 100644
--- a/offlineimap/folder/GmailMaildir.py
+++ b/offlineimap/folder/GmailMaildir.py
@@ -15,13 +15,15 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-
import os
from sys import exc_info
+
from .Maildir import MaildirFolder
from offlineimap import OfflineImapError
import offlineimap.accounts
from offlineimap import imaputil
+from offlineimap.utils import uni
+
class GmailMaildirFolder(MaildirFolder):
"""Folder implementation to support adding labels to messages in a Maildir.
@@ -170,7 +172,9 @@ class GmailMaildirFolder(MaildirFolder):
try:
os.rename(tmppath, filepath)
except OSError as e:
- raise OfflineImapError("Can't rename file '%s' to '%s': %s" % \
+ tmppath = uni.uni2fs(tmppath)
+ filepath = uni.uni2fs(filepath)
+ raise OfflineImapError("Can't rename file '%s' to '%s': %s"%
(tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER), \
None, exc_info()[2]
diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py
index c7e6516..29a2d38 100644
--- a/offlineimap/folder/IMAP.py
+++ b/offlineimap/folder/IMAP.py
@@ -22,8 +22,8 @@ import time
from sys import exc_info
from .Base import BaseFolder
-from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError
-from offlineimap import globals
+from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError, globals
+from offlineimap.utils import uni
from offlineimap.imaplib2 import MonthNames
@@ -66,10 +66,14 @@ class IMAPFolder(BaseFolder):
.. todo: Still valid? Needs verification
:param: Enforce new SELECT even if we are on that folder already.
:returns: raises :exc:`OfflineImapError` severity FOLDER on error"""
+
try:
- imapobj.select(self.getfullname(), force = force)
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname, force=force)
except imapobj.readonly:
- imapobj.select(self.getfullname(), readonly = True, force = force)
+ imapobj.select(fullname, readonly=True, force=force)
# Interface from BaseFolder
def suggeststhreads(self):
@@ -116,8 +120,10 @@ class IMAPFolder(BaseFolder):
imapobj = self.imapserver.acquireconnection()
try:
# Select folder and get number of messages
- restype, imapdata = imapobj.select(self.getfullname(), True,
- True)
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ restype, imapdata = imapobj.select(fullname, True, True)
self.imapserver.releaseconnection(imapobj)
except OfflineImapError as e:
# retry on dropped connections, raise otherwise
@@ -156,7 +162,10 @@ class IMAPFolder(BaseFolder):
Returns: range(s) for messages or None if no messages
are to be fetched."""
- res_type, imapdata = imapobj.select(self.getfullname(), True, True)
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ res_type, imapdata = imapobj.select(fullname, True, True)
if imapdata == [None] or imapdata[0] == '0':
# Empty folder, no need to populate message list
return None
@@ -196,7 +205,8 @@ class IMAPFolder(BaseFolder):
if res_type != 'OK':
raise OfflineImapError("SEARCH in folder [%s]%s failed. "
"Search string was '%s'. Server responded '[%s] %s'"% (
- self.getrepository(), self, search_cond, res_type, res_data),
+ uni.uni2fs(self.getrepository()), self,
+ uni.uni2fs(search_cond), res_type, uni.uni2fs(res_data)),
OfflineImapError.ERROR.FOLDER)
# Resulting MSN are separated by space, coalesce into ranges
@@ -223,10 +233,13 @@ class IMAPFolder(BaseFolder):
# imaplib2 from quoting the sequence.
res_type, response = imapobj.fetch("'%s'"%
msgsToFetch, '(FLAGS UID)')
+ if globals.options.use_unicode:
+ response = uni.imap2uni(response)
if res_type != 'OK':
raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. "
- "Server responded '[%s] %s'"% (self.getrepository(), self,
- res_type, response), OfflineImapError.ERROR.FOLDER)
+ "Server responded '[%s] %s'"% (
+ uni.uni2fs(self.getrepository()), self,
+ res_type, uni.uni2fs(response)), OfflineImapError.ERROR.FOLDER)
finally:
self.imapserver.releaseconnection(imapobj)
@@ -238,9 +251,8 @@ class IMAPFolder(BaseFolder):
messagestr = messagestr.split(' ', 1)[1]
options = imaputil.flags2hash(messagestr)
if not 'UID' in options:
- self.ui.warn('No UID in message with options %s'% \
- str(options),
- minor = 1)
+ self.ui.warn(u'No UID in message with options %s'%
+ str(options), minor=1)
else:
uid = long(options['UID'])
self.messagelist[uid] = self.msglist_item_initializer(uid)
@@ -282,7 +294,7 @@ class IMAPFolder(BaseFolder):
else:
dbg_output = data
- self.ui.debug('imap', "Returned object from fetching %d: '%s'"%
+ self.ui.debug('imap', u"Returned object from fetching %d: '%s'"%
(uid, dbg_output))
return data
@@ -333,21 +345,26 @@ class IMAPFolder(BaseFolder):
headername, headervalue)[1][0]
except imapobj.error as err:
# IMAP server doesn't implement search or had a problem.
- self.ui.debug('imap', "__savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s"% (err, headername))
+ self.ui.debug('imap', u"__savemessage_searchforheader: "
+ "got IMAP error '%s' while attempting to UID SEARCH "
+ "for message with header %s"% (err, headername))
return 0
- self.ui.debug('imap', '__savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
+ self.ui.debug('imap', u'__savemessage_searchforheader got '
+ 'initial matchinguids: %s'% repr(matchinguids))
if matchinguids == '':
- self.ui.debug('imap', "__savemessage_searchforheader: UID SEARCH for message with header %s yielded no results"% headername)
+ self.ui.debug('imap', u"__savemessage_searchforheader: "
+ "UID SEARCH for message with header %s yielded no results"%
+ headername)
return 0
matchinguids = matchinguids.split(' ')
- self.ui.debug('imap', '__savemessage_searchforheader: matchinguids now ' + \
- repr(matchinguids))
+ self.ui.debug('imap', u'__savemessage_searchforheader: '
+ 'matchinguids now %s'% repr(matchinguids))
if len(matchinguids) != 1 or matchinguids[0] == None:
raise ValueError("While attempting to find UID for message with "
- "header %s, got wrong-sized matchinguids of %s"%\
- (headername, str(matchinguids)))
+ "header %s, got wrong-sized matchinguids of %s"%
+ (uni.uni2fs(headername), str(matchinguids)))
return long(matchinguids[0])
def __savemessage_fetchheaders(self, imapobj, headername, headervalue):
@@ -373,8 +390,8 @@ class IMAPFolder(BaseFolder):
Returns UID when found, 0 when not found."""
- self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \
- (headername, headervalue))
+ self.ui.debug('imap', u'__savemessage_fetchheaders called for %s: %s'%
+ (headername, headervalue))
# run "fetch X:* rfc822.header"
# since we stored the mail we are looking for just recently, it would
@@ -395,8 +412,11 @@ class IMAPFolder(BaseFolder):
result = imapobj.uid('FETCH', bytearray('%d:*'% start), 'rfc822.header')
if result[0] != 'OK':
- raise OfflineImapError('Error fetching mail headers: %s'%
- '. '.join(result[1]), OfflineImapError.ERROR.MESSAGE)
+ joint_result = '. '.join(result[1])
+ if globals.options.use_unicode:
+ joint_result = uni.uni2std(uni.imap2uni(joint_result))
+ raise OfflineImapError("Error fetching mail headers: %s"% joint_result,
+ OfflineImapError.ERROR.MESSAGE)
result = result[1]
@@ -413,9 +433,9 @@ class IMAPFolder(BaseFolder):
if uid:
return int(uid.group(1))
else:
- self.ui.warn("Can't parse FETCH response, can't find UID: %s", result.__repr__())
+ self.ui.warn(u"Can't parse FETCH response, can't find UID: %s", result.__repr__())
else:
- self.ui.warn("Can't parse FETCH response, we awaited string: %s", result.__repr__())
+ self.ui.warn(u"Can't parse FETCH response, we awaited string: %s", result.__repr__())
return 0
@@ -471,7 +491,7 @@ class IMAPFolder(BaseFolder):
# or something. Argh. It seems that Time2Internaldate
# will rause a ValueError if the year is 0102 but not 1902,
# but some IMAP servers nonetheless choke on 1902.
- self.ui.debug('imap', "Message with invalid date %s. "
+ self.ui.debug('imap', u"Message with invalid date %s. "
"Server will use local time."% datetuple)
return None
@@ -550,7 +570,7 @@ class IMAPFolder(BaseFolder):
# insert a random unique header that we can fetch later
(headername, headervalue) = self.__generate_randomheader(
content)
- self.ui.debug('imap', 'savemessage: header is: %s: %s'%
+ self.ui.debug('imap', u'savemessage: header is: %s: %s'%
(headername, headervalue))
content = self.addmessageheader(content, CRLF, headername, headervalue)
@@ -558,12 +578,16 @@ class IMAPFolder(BaseFolder):
dbg_output = "%s...%s"% (content[:150], content[-50:])
else:
dbg_output = content
- self.ui.debug('imap', "savemessage: date: %s, content: '%s'"%
+ self.ui.debug('imap', u"savemessage: date: %s, content: '%s'"%
(date, dbg_output))
+ fullname = self.getfullname()
+
try:
# Select folder for append and make the box READ-WRITE
- imapobj.select(self.getfullname())
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname)
except imapobj.readonly:
# readonly exception. Return original uid to notify that
# we did not save the message. (see savemessage in Base.py)
@@ -572,7 +596,7 @@ class IMAPFolder(BaseFolder):
#Do the APPEND
try:
- (typ, dat) = imapobj.append(self.getfullname(),
+ (typ, dat) = imapobj.append(fullname,
imaputil.flagsmaildir2imap(flags), date, content)
# This should only catch 'NO' responses since append()
# will raise an exception for 'BAD' responses:
@@ -596,11 +620,11 @@ class IMAPFolder(BaseFolder):
imapobj = self.imapserver.acquireconnection()
if not retry_left:
raise OfflineImapError("Saving msg (%s) in folder '%s', "
- "repository '%s' failed (abort). Server responded: %s\n"
- "Message content was: %s"%
- (msg_id, self, self.getrepository(), str(e), dbg_output),
- OfflineImapError.ERROR.MESSAGE), \
- None, exc_info()[2]
+ "repository '%s' failed (abort). Server responded: %s\n"
+ "Message content was: %s"% (msg_id, self,
+ uni.uni2fs(self.getrepository()), str(e),
+ uni.uni2fs(dbg_output)), OfflineImapError.ERROR.MESSAGE), \
+ None, exc_info()[2]
# XXX: is this still needed?
self.ui.error(e, exc_info()[2])
except imapobj.error as e: # APPEND failed
@@ -611,8 +635,9 @@ class IMAPFolder(BaseFolder):
imapobj = None
raise OfflineImapError("Saving msg (%s) folder '%s', repo '%s'"
"failed (error). Server responded: %s\nMessage content was: "
- "%s" % (msg_id, self, self.getrepository(), str(e), dbg_output),
- OfflineImapError.ERROR.MESSAGE), None, exc_info()[2]
+ "%s"% (msg_id, self,
+ uni.uni2fs(self.getrepository()), str(e), dbg_output),
+ OfflineImapError.ERROR.MESSAGE), None, exc_info()[2]
# Checkpoint. Let it write out stuff, etc. Eg searches for
# just uploaded messages won't work if we don't do this.
(typ,dat) = imapobj.check()
@@ -656,7 +681,7 @@ class IMAPFolder(BaseFolder):
self.messagelist[uid] = self.msglist_item_initializer(uid)
self.messagelist[uid]['flags'] = flags
- self.ui.debug('imap', 'savemessage: returning new UID %d'% uid)
+ self.ui.debug('imap', u'savemessage: returning new UID %d'% uid)
return uid
@@ -674,7 +699,10 @@ class IMAPFolder(BaseFolder):
fails_left = retry_num # retry on dropped connection
while fails_left:
try:
- imapobj.select(self.getfullname(), readonly = True)
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname, readonly = True)
res_type, data = imapobj.uid('fetch', uids, query)
fails_left = 0
except imapobj.abort as e:
@@ -690,12 +718,13 @@ class IMAPFolder(BaseFolder):
#IMAP server says bad request or UID does not exist
severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to fetch messages UID '%s'."\
- "Server responded: %s %s"% (self.getrepository(), uids,
- res_type, data)
+ "Server responded: %s %s"% (
+ uni.uni2fs(self.getrepository()),uids, res_type, data)
if data == [None]:
#IMAP server did not find a message with this UID
reason = "IMAP server '%s' does not have a message "\
- "with UID '%s'" % (self.getrepository(), uids)
+ "with UID '%s'"% (
+ uni.uni2fs(self.getrepository()), uids)
raise OfflineImapError(reason, severity)
return data
@@ -710,13 +739,17 @@ class IMAPFolder(BaseFolder):
- field: field name to be stored/updated
- data: field contents
"""
- imapobj.select(self.getfullname())
+
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname)
res_type, retdata = imapobj.uid('store', uid, field, data)
if res_type != 'OK':
severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to store %s for message UID '%d'."\
- "Server responded: %s %s"% (
- self.getrepository(), field, uid, res_type, retdata)
+ "Server responded: %s %s"% (uni.uni2fs(self.getrepository()),
+ uni.uni2fs(field), uid, res_type, uni.uni2fs(retdata))
raise OfflineImapError(reason, severity)
return retdata[0]
@@ -771,7 +804,10 @@ class IMAPFolder(BaseFolder):
imapobj = self.imapserver.acquireconnection()
try:
try:
- imapobj.select(self.getfullname())
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname)
except imapobj.readonly:
self.ui.flagstoreadonly(self, uidlist, flags)
return
@@ -845,7 +881,10 @@ class IMAPFolder(BaseFolder):
imapobj = self.imapserver.acquireconnection()
try:
try:
- imapobj.select(self.getfullname())
+ fullname = self.getfullname()
+ if globals.options.use_unicode:
+ fullname = uni.uni2imap(fullname)
+ imapobj.select(fullname)
except imapobj.readonly:
self.ui.deletereadonly(self, uidlist)
return
diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py
index c753f63..e5e245e 100644
--- a/offlineimap/folder/LocalStatus.py
+++ b/offlineimap/folder/LocalStatus.py
@@ -19,6 +19,8 @@ from sys import exc_info
import os
import threading
+from offlineimap import globals
+from offlineimap.utils import uni
from .Base import BaseFolder
@@ -32,7 +34,16 @@ class LocalStatusFolder(BaseFolder):
self.sep = '.' #needs to be set before super.__init__()
super(LocalStatusFolder, self).__init__(name, repository)
self.root = repository.root
- self.filename = os.path.join(self.getroot(), self.getfolderbasename())
+ folderbasename = self.getfolderbasename()
+ root = self.getroot()
+ if globals.options.use_unicode:
+ root = uni.uni2fs(root)
+ # Fix the filename to IMAP UTF-7 encoding. This prevent from playing
+ # with multiple cache files representative of the same folder.
+ folderbasename = uni.uni2fs(uni.uni2imap(folderbasename))
+ self.filename = uni.uni2fs(os.path.join(root, folderbasename))
+ else:
+ self.filename = os.path.join(root, folderbasename)
self.messagelist = {}
self.savelock = threading.Lock()
# Should we perform fsyncs as often as possible?
@@ -72,7 +83,13 @@ class LocalStatusFolder(BaseFolder):
uid = long(uid)
flags = set(flags)
except ValueError as e:
- errstr = "Corrupt line '%s' in cache file '%s'" % \
+ # Mixing encoding:
+ # - line: should be ASCII but unexpected bug could make it
+ # string of bytes with unkown encoding.
+ # - self.filename: filesystem encoded.
+ # Converting from unkown encoding is worse than keeping things
+ # as-is.
+ errstr = "Corrupt line '%s' in cache file '%s'"% \
(line, self.filename)
self.ui.warn(errstr)
raise ValueError(errstr), None, exc_info()[2]
@@ -95,9 +112,17 @@ class LocalStatusFolder(BaseFolder):
mtime = long(mtime)
labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0])
except ValueError as e:
+ # Mixing encoding:
+ # - line: should be ASCII but unexpected bug could make it
+ # string of bytes with unkown encoding.
+ # - self.filename: filesystem encoded.
+ # Converting from unkown encoding is worse than keeping things
+ # as-is.
errstr = "Corrupt line '%s' in cache file '%s'"% \
(line, self.filename)
self.ui.warn(errstr)
+ if globals.options.use_unicode:
+ errstr = uni.uni2std(errstr)
raise ValueError(errstr), None, exc_info()[2]
self.messagelist[uid] = self.msglist_item_initializer(uid)
self.messagelist[uid]['flags'] = flags
@@ -123,7 +148,7 @@ class LocalStatusFolder(BaseFolder):
# Convert from format v1.
elif line == (self.magicline % 1):
- self.ui._msg('Upgrading LocalStatus cache from version 1'
+ self.ui._msg(u'Upgrading LocalStatus cache from version 1'
'to version 2 for %s:%s'% (self.repository, self))
self.readstatus_v1(cachefd)
cachefd.close()
@@ -139,14 +164,15 @@ class LocalStatusFolder(BaseFolder):
# Something is wrong.
else:
- errstr = "Unrecognized cache magicline in '%s'" % self.filename
+ # Keep string filesystem encoded.
+ errstr = "Unrecognized cache magicline in '%s'"% self.filename
self.ui.warn(errstr)
raise ValueError(errstr)
if not line:
# The status file is empty - should not have happened,
# but somehow did.
- errstr = "Cache file '%s' is empty."% self.filename
+ errstr = u"Cache file '%s' is empty."% self.filename
self.ui.warn(errstr)
cachefd.close()
return
@@ -172,7 +198,11 @@ class LocalStatusFolder(BaseFolder):
for msg in self.messagelist.values():
flags = ''.join(sorted(msg['flags']))
labels = ', '.join(sorted(msg['labels']))
- cachefd.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels))
+ data = u"%s|%s|%d|%s\n"% (msg['uid'], flags, msg['mtime'], labels)
+ # Ensure all data is full ASCII.
+ uni.uni2str(data,
+ exception_msg="unsupported character in flags or labels")
+ cachefd.write(data)
cachefd.flush()
if self.doautosave:
os.fsync(cachefd.fileno())
@@ -180,7 +210,10 @@ class LocalStatusFolder(BaseFolder):
os.rename(self.filename + ".tmp", self.filename)
if self.doautosave:
- fd = os.open(os.path.dirname(self.filename), os.O_RDONLY)
+ # fsync on directory ensure that a newly created file will exist
+ # in the directory. The filesystem must support this feature.
+ p = os.path.dirname(self.filename)
+ fd = os.open(p, os.O_RDONLY)
os.fsync(fd)
os.close(fd)
diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py
index 8a3b9df..dc36953 100644
--- a/offlineimap/folder/LocalStatusSQLite.py
+++ b/offlineimap/folder/LocalStatusSQLite.py
@@ -22,6 +22,8 @@ try:
except:
pass #fail only if needed later on, not on import
+from offlineimap import globals
+from offlineimap.utils import uni
from .Base import BaseFolder
@@ -47,7 +49,12 @@ class LocalStatusSQLiteFolder(BaseFolder):
self.sep = '.' #needs to be set before super.__init__()
super(LocalStatusSQLiteFolder, self).__init__(name, repository)
self.root = repository.root
- self.filename = os.path.join(self.getroot(), self.getfolderbasename())
+ folderbasename = self.getfolderbasename()
+ if globals.options.use_unicode:
+ folderbasename = uni.uni2imap(folderbasename)
+ self.filename = uni.uni2fs(os.path.join(self.getroot(), folderbasename))
+ else:
+ self.filename = os.path.join(self.getroot(), folderbasename)
self.messagelist = {}
self._newfolder = False # flag if the folder is new
@@ -56,6 +63,7 @@ class LocalStatusSQLiteFolder(BaseFolder):
if not os.path.exists(dirname):
os.makedirs(dirname)
if not os.path.isdir(dirname):
+ # dirname is expected filesystem encoded.
raise UserWarning("SQLite database path '%s' is not a directory."%
dirname)
@@ -172,8 +180,9 @@ class LocalStatusSQLiteFolder(BaseFolder):
self.connection must point to the opened and valid SQlite
database connection."""
- self.ui._msg('Creating new Local Status db for %s:%s' \
- % (self.repository, self))
+
+ self.ui._msg(u'Creating new Local Status db for %s:%s'%
+ (self.repository.getname(), self.getname()))
self.connection.executescript("""
CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128));
INSERT INTO metadata VALUES('db_version', '2');
diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index b35bfc2..795ccfc 100644
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -31,7 +31,9 @@ try: # python 2.6 has set() built in
except NameError:
from sets import Set as set
+from offlineimap import globals
from offlineimap import OfflineImapError
+from offlineimap.utils import uni
# Find the UID in a message filename
re_uidmatch = re.compile(',U=(\d+)')
@@ -74,7 +76,7 @@ class MaildirFolder(BaseFolder):
# Everything up to the first comma or colon (or ! if Windows):
self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)')
#folder's md, so we can match with recorded file md5 for validity
- self._foldermd5 = md5(self.getvisiblename()).hexdigest()
+ self._foldermd5 = md5(uni.uni2imap(self.getvisiblename())).hexdigest()
# Cache the full folder path, as we use getfullname() very often
self._fullname = os.path.join(self.getroot(), self.getname())
@@ -279,8 +281,10 @@ class MaildirFolder(BaseFolder):
while tries:
tries = tries - 1
try:
- fd = os.open(os.path.join(self.getfullname(), tmpname),
- os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666)
+ path = os.path.join(self.getfullname(), tmpname)
+ if globals.options.use_unicode:
+ path = uni.uni2fs(path)
+ fd = os.open(path, os.O_EXCL|os.O_CREAT|os.O_WRONLY, 0o666)
break
except OSError as e:
if e.errno == e.EEXIST:
@@ -288,12 +292,15 @@ class MaildirFolder(BaseFolder):
time.sleep(0.23)
continue
severity = OfflineImapError.ERROR.MESSAGE
- raise OfflineImapError("Unique filename %s already exists." % \
+ # filename is expected filesystem encoded.
+ raise OfflineImapError("Unique filename %s already exists."%
filename, severity), None, exc_info()[2]
else:
raise
fd = os.fdopen(fd, 'wt')
+ if globals.options.use_unicode:
+ content = uni.uni2bytes(content)
fd.write(content)
# Make sure the data hits the disk
fd.flush()
@@ -336,7 +343,7 @@ class MaildirFolder(BaseFolder):
self.messagelist[uid]['filename'] = tmpname
# savemessageflags moves msg to 'cur' or 'new' as appropriate
self.savemessageflags(uid, flags)
- self.ui.debug('maildir', 'savemessage: returning uid %d' % uid)
+ self.ui.debug('maildir', u'savemessage: returning uid %d'% uid)
return uid
# Interface from BaseFolder
@@ -374,13 +381,21 @@ class MaildirFolder(BaseFolder):
newfilename = os.path.join(dir_prefix, filename)
if (newfilename != oldfilename):
try:
- os.rename(os.path.join(self.getfullname(), oldfilename),
- os.path.join(self.getfullname(), newfilename))
+ old = os.path.join(self.getfullname(), oldfilename)
+ new = os.path.join(self.getfullname(), newfilename)
+ if globals.options.use_unicode:
+ old = uni.uni2fs(old)
+ new = uni.uni2fs(new)
+ os.rename(old, new)
except OSError as e:
- raise OfflineImapError("Can't rename file '%s' to '%s': %s" % (
- oldfilename, newfilename, e[1]),
- OfflineImapError.ERROR.FOLDER), \
- None, exc_info()[2]
+ if globals.options.use_unicode:
+ oldfilename = uni.uni2fs(oldfilename)
+ newfilename = uni.uni2fs(newfilename)
+ # File names are expected filesystem encoded.
+ raise OfflineImapError("Can't rename file '%s' to '%s': %s"% (
+ oldfilename, newfilename, e[1]),
+ OfflineImapError.ERROR.FOLDER), \
+ None, exc_info()[2]
self.messagelist[uid]['flags'] = flags
self.messagelist[uid]['filename'] = newfilename
@@ -403,9 +418,13 @@ class MaildirFolder(BaseFolder):
dir_prefix, filename = os.path.split(oldfilename)
flags = self.getmessageflags(uid)
newfilename = os.path.join(dir_prefix,
- self.new_message_filename(new_uid, flags))
- os.rename(os.path.join(self.getfullname(), oldfilename),
- os.path.join(self.getfullname(), newfilename))
+ self.new_message_filename(new_uid, flags))
+ old = os.path.join(self.getfullname(), oldfilename)
+ new = os.path.join(self.getfullname(), newfilename)
+ if globals.options.use_unicode:
+ old = uni.uni2fs(old)
+ new = uni.uni2fs(new)
+ os.rename(old, new)
self.messagelist[new_uid] = self.messagelist[uid]
self.messagelist[new_uid]['filename'] = newfilename
del self.messagelist[uid]
@@ -419,11 +438,16 @@ class MaildirFolder(BaseFolder):
:return: Nothing, or an Exception if UID but no corresponding file
found.
"""
+
if not self.uidexists(uid):
return
filename = self.messagelist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
+ if globals.options.use_unicode:
+ filepath = uni.uni2fs(filepath, exception_msg=
+ "unsupported character in message path: %s"%
+ (uni.uni2std(filepath)))
try:
os.unlink(filepath)
except OSError:
@@ -432,6 +456,9 @@ class MaildirFolder(BaseFolder):
if uid in newmsglist: # Nope, try new filename.
filename = newmsglist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
+ if globals.options.use_unicode:
+ filepath = uni.uni2fs(filepath, exception_msg=
+ "unsupported character in message path: %s"% (filepath))
os.unlink(filepath)
# Yep -- return.
del(self.messagelist[uid])
diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py
index e8ca9a7..7119e0c 100644
--- a/offlineimap/folder/UIDMaps.py
+++ b/offlineimap/folder/UIDMaps.py
@@ -16,10 +16,14 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
from sys import exc_info
+import os.path
+import codecs
from threading import Lock
+
from offlineimap import OfflineImapError
from .IMAP import IMAPFolder
-import os.path
+from offlineimap.utils import uni
+from offlineimap import globals
class MappedIMAPFolder(IMAPFolder):
"""IMAP class to map between Folder() instances where both side assign a uid
@@ -61,6 +65,11 @@ class MappedIMAPFolder(IMAPFolder):
try:
line = line.strip()
except ValueError:
+ # Mixing encodings.
+ # - line: should be ASCII but unexpected bug could turn it
+ # encoded with unkown encoding.
+ # - mapfilename: filesystem encoded.
+ # Keeping variables as string of bytes.
raise Exception("Corrupt line '%s' in UID mapping file '%s'"%
(line, mapfilename)), None, exc_info()[2]
(str1, str2) = line.split(':')
@@ -76,11 +85,21 @@ class MappedIMAPFolder(IMAPFolder):
mapfilename = self._getmapfilename()
if dolock: self.maplock.acquire()
try:
- file = open(mapfilename + ".tmp", 'wt')
+ tmpname = mapfilename + ".tmp"
+ if globals.options.use_unicode:
+ tmpname = uni.uni2fs(tmpname)
+ file = codecs.open(tmpname, 'wt', uni.ENCODING)
+ else:
+ file = open(tmpname, 'wt')
for (key, value) in self.diskl2r.iteritems():
- file.write("%d:%d\n"% (key, value))
+ data = "%d:%d\n"% (key, value)
+ if globals.options.use_unicode:
+ data = uni.uni2bytes(data)
+ file.write(data)
file.close()
- os.rename(mapfilename + '.tmp', mapfilename)
+ if globals.options.use_unicode:
+ mapfilename = uni.uni2fs(mapfilename)
+ os.rename(tmpname, mapfilename)
finally:
if dolock: self.maplock.release()
diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py
index 83ffe9a..a3c2f62 100644
--- a/offlineimap/imaplibutil.py
+++ b/offlineimap/imaplibutil.py
@@ -25,6 +25,8 @@ from hashlib import sha1
from offlineimap.ui import getglobalui
from offlineimap import OfflineImapError
from offlineimap.imaplib2 import IMAP4, IMAP4_SSL, zlib, InternalDate, Mon2num
+from offlineimap import globals
+from offlineimap.utils import uni
class UsefulIMAPMixIn(object):
@@ -54,13 +56,13 @@ class UsefulIMAPMixIn(object):
except self.abort as e:
# self.abort is raised when we are supposed to retry
errstr = "Server '%s' closed connection, error on SELECT '%s'. Ser"\
- "ver said: %s" % (self.host, mailbox, e.args[0])
+ "ver said: %s"% (self.host, uni.uni2fs(mailbox), e.args[0])
severity = OfflineImapError.ERROR.FOLDER_RETRY
raise OfflineImapError(errstr, severity), None, exc_info()[2]
if result[0] != 'OK':
#in case of error, bail out with OfflineImapError
- errstr = "Error SELECTing mailbox '%s', server reply:\n%s" %\
- (mailbox, result)
+ errstr = "Error SELECTing mailbox '%s', server reply:\n%s"%\
+ (uni.uni2fs(mailbox), uni.uni2fs(result))
severity = OfflineImapError.ERROR.FOLDER
raise OfflineImapError(errstr, severity)
return result
@@ -117,6 +119,10 @@ class IMAP4_Tunnel(UsefulIMAPMixIn, IMAP4):
return self.decompressor.decompress(data, size)
def send(self, data):
+ #FIXME UNICODE: delete
+ if globals.options.use_unicode:
+ data = uni.uni2imap(data, "cannot send: "
+ "unsupported character in %s"% data)
if self.compressor is not None:
data = self.compressor.compress(data)
data += self.compressor.flush(zlib.Z_SYNC_FLUSH)
@@ -134,7 +140,8 @@ def new_mesg(self, s, tn=None, secs=None):
if tn is None:
tn = threading.currentThread().getName()
tm = time.strftime('%M:%S', time.localtime(secs))
- getglobalui().debug('imap', ' %s.%02d %s %s' % (tm, (secs*100)%100, tn, s))
+ getglobalui().debug(u'imap', u' %s.%02d %s %s'%
+ (tm, (secs*100)%100, tn, s))
class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL):
@@ -151,21 +158,23 @@ class WrappedIMAP4_SSL(UsefulIMAPMixIn, IMAP4_SSL):
def open(self, host=None, port=None):
if not self.ca_certs and not self._fingerprint:
raise OfflineImapError("No CA certificates "
- "and no server fingerprints configured. "
- "You must configure at least something, otherwise "
- "having SSL helps nothing.", OfflineImapError.ERROR.REPO)
+ "and no server fingerprints configured. "
+ "You must configure at least something, otherwise "
+ "having SSL helps nothing.",
+ OfflineImapError.ERROR.REPO)
super(WrappedIMAP4_SSL, self).open(host, port)
if self._fingerprint:
# compare fingerprints
fingerprint = sha1(self.sock.getpeercert(True)).hexdigest()
if fingerprint not in self._fingerprint:
raise OfflineImapError("Server SSL fingerprint '%s' "
- "for hostname '%s' "
- "does not match configured fingerprint(s) %s. "
- "Please verify and set 'cert_fingerprint' accordingly "
- "if not set yet."%
- (fingerprint, host, self._fingerprint),
- OfflineImapError.ERROR.REPO)
+ "for hostname '%s' "
+ "does not match configured fingerprint(s) %s. "
+ "Please verify and set 'cert_fingerprint' accordingly "
+ "if not set yet."%
+ (uni.uni2fs(fingerprint), host,
+ uni.uni2fs(self._fingerprint)),
+ OfflineImapError.ERROR.REPO)
class WrappedIMAP4(UsefulIMAPMixIn, IMAP4):
diff --git a/offlineimap/imapserver.py b/offlineimap/imapserver.py
index 3d69426..3b168bd 100644
--- a/offlineimap/imapserver.py
+++ b/offlineimap/imapserver.py
@@ -15,10 +15,6 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError
-from offlineimap.ui import getglobalui
-from threading import Lock, BoundedSemaphore, Thread, Event, currentThread
-import offlineimap.accounts
import hmac
import socket
import base64
@@ -28,6 +24,12 @@ from sys import exc_info
from socket import gaierror
from ssl import SSLError, cert_time_to_seconds
+from offlineimap import imaplibutil, imaputil, threadutil, OfflineImapError, globals
+from offlineimap.ui import getglobalui
+from offlineimap.utils import uni
+from threading import Lock, BoundedSemaphore, Thread, Event, currentThread
+import offlineimap.accounts
+
try:
# do we have a recent pykerberos?
have_gss = False
@@ -85,6 +87,8 @@ class IMAPServer:
if self.sslcacertfile is None:
self.__verifycert = None # disable cert verification
self.fingerprint = repos.get_ssl_fingerprint()
+ if globals.options.use_unicode:
+ self.fingerprint = uni.uni2str(self.fingerprint)
self.sslversion = repos.getsslversion()
self.delim = None
@@ -143,17 +147,17 @@ class IMAPServer:
def __md5handler(self, response):
challenge = response.strip()
- self.ui.debug('imap', '__md5handler: got challenge %s'% challenge)
+ self.ui.debug('imap', u'__md5handler: got challenge %s'% challenge)
passwd = self.__getpassword()
retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
- self.ui.debug('imap', '__md5handler: returning %s'% retval)
+ self.ui.debug('imap', u'__md5handler: returning %s'% retval)
return retval
def __loginauth(self, imapobj):
""" Basic authentication via LOGIN command."""
- self.ui.debug('imap', 'Attempting IMAP LOGIN authentication')
+ self.ui.debug('imap', u'Attempting IMAP LOGIN authentication')
imapobj.login(self.username, self.__getpassword())
@@ -168,7 +172,7 @@ class IMAPServer:
authz = self.user_identity
NULL = u'\x00'
retval = NULL.join((authz, authc, passwd)).encode('utf-8')
- self.ui.debug('imap', '__plainhandler: returning %s' % retval)
+ self.ui.debug('imap', u'__plainhandler: returning %s'% retval)
return retval
@@ -193,7 +197,7 @@ class IMAPServer:
except kerberos.GSSError as err:
# Kerberos errored out on us, respond with None to cancel the
# authentication
- self.ui.debug('imap', '%s: %s'% (err[0][0], err[1][0]))
+ self.ui.debug('imap', u'%s: %s'% (err[0][0], err[1][0]))
return None
if not response:
@@ -203,7 +207,7 @@ class IMAPServer:
def __start_tls(self, imapobj):
if 'STARTTLS' in imapobj.capabilities and not self.usessl:
- self.ui.debug('imap', 'Using STARTTLS connection')
+ self.ui.debug('imap', u'Using STARTTLS connection')
try:
imapobj.starttls()
except imapobj.error as e:
@@ -304,8 +308,8 @@ class IMAPServer:
for m in mechs:
if m not in auth_methods:
- raise Exception("Bad authentication method %s, "
- "please, file OfflineIMAP bug" % m)
+ raise Exception(u"Bad authentication method %s, "
+ "please, file OfflineIMAP bug"% uni.uni2fs(m))
func, tryTLS, check_cap = auth_methods[m]
@@ -321,14 +325,13 @@ class IMAPServer:
continue
tried_to_authn = True
- self.ui.debug('imap', u'Attempting '
- '%s authentication'% m)
+ self.ui.debug('imap', u'Attempting %s authentication'% m)
try:
if func(imapobj):
return
except (imapobj.error, OfflineImapError) as e:
- self.ui.warn('%s authentication failed: %s'% (m, e))
- exc_stack.append((m, e))
+ self.ui.warn(u'%s authentication failed: %s'% (m, e))
+ exc_stack.append((uni.uni2fs(m), e))
if len(exc_stack):
msg = "\n\t".join(map(
@@ -345,8 +348,9 @@ class IMAPServer:
))
raise OfflineImapError(u"Repository %s: no supported "
"authentication mechanisms found; configured %s, "
- "server advertises %s"% (self.repos,
- ", ".join(self.authmechs), methods),
+ "server advertises %s"% (uni.uni2fs(self.repos),
+ ", ".join(uni.uni2fs(self.authmechs)),
+ uni.uni2fs(methods)),
OfflineImapError.ERROR.REPO)
@@ -439,10 +443,10 @@ class IMAPServer:
# No Folders were returned. This occurs, e.g. if the
# 'reference' prefix does not exist on the mail
# server. Raise exception.
- err = "Server '%s' returned no folders in '%s'"% \
+ err = u"Server '%s' returned no folders in '%s'"% \
(self.repos.getname(), self.reference)
self.ui.warn(err)
- raise Exception(err)
+ raise Exception(uni.uni2fs(err))
self.delim, self.root = \
imaputil.imapsplit(listres[0])[1:]
self.delim = imaputil.dequote(self.delim)
@@ -466,7 +470,7 @@ class IMAPServer:
reason = "Could not resolve name '%s' for repository "\
"'%s'. Make sure you have configured the ser"\
"ver name correctly and that you are online."%\
- (self.hostname, self.repos)
+ (uni.uni2fs(self.hostname), uni.uni2fs(self.repos))
raise OfflineImapError(reason, severity), None, exc_info()[2]
elif isinstance(e, SSLError) and e.errno == errno.EPERM:
@@ -478,8 +482,8 @@ class IMAPServer:
" to the correct port."% (self.hostname, self.port)
else:
reason = "Unknown SSL protocol connecting to host '%s' for "\
- "repository '%s'. OpenSSL responded:\n%s"\
- % (self.hostname, self.repos, e)
+ "repository '%s'. OpenSSL responded:\n%s"%\
+ (uni.uni2fs(self.hostname), uni.uni2fs(self.repos), e)
raise OfflineImapError(reason, severity), None, exc_info()[2]
elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED:
@@ -494,8 +498,8 @@ class IMAPServer:
# socket.error(last_error) raised
if str(e)[:24] == "can't open socket; error":
raise OfflineImapError("Could not connect to remote server '%s' "\
- "for repository '%s'. Remote does not answer."
- % (self.hostname, self.repos),
+ "for repository '%s'. Remote does not answer."%
+ (uni.uni2fs(self.hostname), uni.uni2fs(self.repos)),
OfflineImapError.ERROR.REPO), None, exc_info()[2]
else:
# re-raise all other errors
@@ -541,7 +545,7 @@ class IMAPServer:
is expected to be invoked in a separate thread, which should be join()'d
after the event is set."""
- self.ui.debug('imap', 'keepalive thread started')
+ self.ui.debug('imap', u'keepalive thread started')
while not event.isSet():
self.connectionlock.acquire()
numconnections = len(self.assignedconnections) + \
@@ -550,7 +554,8 @@ class IMAPServer:
threads = []
for i in range(numconnections):
- self.ui.debug('imap', 'keepalive: processing connection %d of %d'% (i, numconnections))
+ self.ui.debug('imap', u'keepalive: '
+ 'processing connection %d of %d'% (i, numconnections))
if len(self.idlefolders) > i:
# IDLE thread
idler = IdleThread(self, self.idlefolders[i])
@@ -560,16 +565,16 @@ class IMAPServer:
idler.start()
threads.append(idler)
- self.ui.debug('imap', 'keepalive: waiting for timeout')
+ self.ui.debug('imap', u'keepalive: waiting for timeout')
event.wait(timeout)
- self.ui.debug('imap', 'keepalive: after wait')
+ self.ui.debug('imap', u'keepalive: after wait')
for idler in threads:
# Make sure all the commands have completed.
idler.stop()
idler.join()
- self.ui.debug('imap', 'keepalive: all threads joined')
- self.ui.debug('imap', 'keepalive: event is set; exiting')
+ self.ui.debug('imap', u'keepalive: all threads joined')
+ self.ui.debug('imap', u'keepalive: event is set; exiting')
return
def __verifycert(self, cert, hostname):
@@ -578,9 +583,9 @@ class IMAPServer:
CRLs are not handled.
Returns error message if any problems are found and None on success."""
- errstr = "CA Cert verifying failed: "
+ errstr = u"CA Cert verifying failed: "
if not cert:
- return ('%s no certificate received'% errstr)
+ return (u'%s no certificate received'% errstr)
dnsname = hostname.lower()
certnames = []
@@ -588,7 +593,7 @@ class IMAPServer:
notafter = cert.get('notAfter')
if notafter:
if time.time() >= cert_time_to_seconds(notafter):
- return '%s certificate expired %s'% (errstr, notafter)
+ return u'%s certificate expired %s'% (errstr, notafter)
# First read commonName
for s in cert.get('subject', []):
@@ -609,7 +614,7 @@ class IMAPServer:
'.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]):
return None
- return ('%s no matching domain name found in certificate'% errstr)
+ return (u'%s no matching domain name found in certificate'% errstr)
class IdleThread(object):
@@ -648,7 +653,7 @@ class IdleThread(object):
try:
imapobj.noop()
except imapobj.abort:
- self.ui.warn('Attempting NOOP on dropped connection %s'%
+ self.ui.warn(u'Attempting NOOP on dropped connection %s'%
imapobj.identifier)
self.parent.releaseconnection(imapobj, True)
imapobj = None
@@ -696,7 +701,10 @@ class IdleThread(object):
while not success:
imapobj = self.parent.acquireconnection()
try:
- imapobj.select(self.folder)
+ folder = self.folder
+ if globals.options.use_unicode:
+ folder = uni.uni2imap(folder)
+ imapobj.select(folder)
except OfflineImapError as e:
if e.severity == OfflineImapError.ERROR.FOLDER_RETRY:
# Connection closed, release connection and retry
@@ -717,7 +725,7 @@ class IdleThread(object):
# End IDLE mode with noop, imapobj can point to a dropped conn.
imapobj.noop()
except imapobj.abort:
- self.ui.warn('Attempting NOOP on dropped connection %s'%
+ self.ui.warn(u'Attempting NOOP on dropped connection %s'%
imapobj.identifier)
self.parent.releaseconnection(imapobj, True)
else:
diff --git a/offlineimap/imaputil.py b/offlineimap/imaputil.py
index f1f287b..e1ce55d 100644
--- a/offlineimap/imaputil.py
+++ b/offlineimap/imaputil.py
@@ -17,7 +17,9 @@
import re
import string
+
from offlineimap.ui import getglobalui
+from offlineimap.utils import uni
## Globals
@@ -30,7 +32,7 @@ def __debug(*args):
msg = []
for arg in args:
msg.append(str(arg))
- getglobalui().debug('imap', " ".join(msg))
+ getglobalui().debug('imap', u" ".join(msg))
def dequote(s):
"""Takes string which may or may not be quoted and unquotes it.
@@ -63,7 +65,7 @@ def flagsplit(s):
"""
if s[0] != '(' or s[-1] != ')':
- raise ValueError("Passed s '%s' is not a flag list"% s)
+ raise ValueError("Passed s '%s' is not a flag list"% uni.uni2fs(s))
return imapsplit(s[1:-1])
def __options2hash(list):
@@ -77,7 +79,7 @@ def __options2hash(list):
while (counter < len(list)):
retval[list[counter]] = list[counter + 1]
counter += 2
- __debug("__options2hash returning:", retval)
+ __debug(u"__options2hash returning:", retval)
return retval
def flags2hash(flags):
@@ -99,7 +101,7 @@ def imapsplit(imapstring):
['(\\HasNoChildren)', '"."', '"INBOX.Sent"']"""
if not isinstance(imapstring, basestring):
- __debug("imapsplit() got a non-string input; working around.")
+ __debug(u"imapsplit() got a non-string input; working around.")
# Sometimes, imaplib will throw us a tuple if the input
# contains a literal. See Python bug
# #619732 at https://sourceforge.net/tracker/index.php?func=detail&aid=619732&group_id=5470&atid=105470
@@ -121,7 +123,7 @@ def imapsplit(imapstring):
arg = arg.replace('\\', '\\\\')
arg = arg.replace('"', '\\"')
arg = '"%s"' % arg
- __debug("imapsplit() non-string [%d]: Appending %s"% (i, arg))
+ __debug(u"imapsplit() non-string [%d]: Appending %s"% (i, arg))
retval.append(arg)
else:
# Even -- we have a string that ends with a literal
@@ -130,10 +132,10 @@ def imapsplit(imapstring):
# Recursion to the rescue.
arg = imapstring[i]
arg = re.sub('\{\d+\}$', '', arg)
- __debug("imapsplit() non-string [%d]: Feeding %s to recursion"%\
+ __debug(u"imapsplit() non-string [%d]: Feeding %s to recursion"%
(i, arg))
retval.extend(imapsplit(arg))
- __debug("imapsplit() non-string: returning %s" % str(retval))
+ __debug(u"imapsplit() non-string: returning %s"% str(retval))
return retval
workstr = imapstring.strip()
@@ -254,7 +256,8 @@ def __split_quoted(s):
while True:
next_q = rest.find(q)
if next_q == -1:
- raise ValueError("can't find ending quote '%s' in '%s'"% (q, s))
+ raise ValueError("can't find ending quote '%s' in '%s'"%
+ (q, uni.uni2fs(s)))
# If quote is preceeded by even number of backslashes,
# then it is the ending quote, otherwise the quote
# character is escaped by backslash, so we should
diff --git a/offlineimap/init.py b/offlineimap/init.py
index 7f6a679..0f267ba 100644
--- a/offlineimap/init.py
+++ b/offlineimap/init.py
@@ -22,14 +22,15 @@ import offlineimap.imaplib2 as imaplib
import signal
import socket
import logging
-from optparse import OptionParser
+import codecs
+from optparse import OptionParser, SUPPRESS_HELP
import offlineimap
from offlineimap import accounts, threadutil, syncmaster
from offlineimap import globals
from offlineimap.ui import UI_LIST, setglobalui, getglobalui
from offlineimap.CustomConfig import CustomConfigParser
-from offlineimap.utils import stacktrace
+from offlineimap.utils import stacktrace, uni
class OfflineImap:
@@ -82,6 +83,15 @@ class OfflineImap:
"maxsyncaccounts and all maxconnections configuration file "
"variables to 1.")
+ parser.add_option("--unicode", dest="unicode_help",
+ action='store_true',
+ help="Enable Unicode support (EXPERIMENTAL).")
+ parser.add_option("--enable-unicode",
+ action='store_true', dest='use_unicode', help=SUPPRESS_HELP)
+ parser.add_option("--no-unicode",
+ action='store_false', dest='use_unicode',
+ help="Disable Unicode support (default).")
+
parser.add_option("-P", dest="profiledir", metavar="DIR",
help="Sets OfflineIMAP into profile mode. The program "
"will create DIR (it must not already exist). "
@@ -92,7 +102,8 @@ class OfflineImap:
"specific reason to do so. It will significantly "
"decrease program performance, may reduce reliability, "
"and can generate huge amounts of data. This option "
- "implies the -1 option.")
+ "implies the -1 option. "
+ "Support only ASCII encoded path.")
parser.add_option("-a", dest="accounts", metavar="ACCOUNTS",
help="Overrides the accounts section in the config file. "
@@ -123,6 +134,7 @@ class OfflineImap:
help="Log to FILE")
parser.add_option("-f", dest="folders", metavar="folder1,[folder2...]",
+ type=str,
help="Only sync the specified folders. The folder names "
"are the *untranslated* foldernames of the remote repository. "
"This command-line option overrides any 'folderfilter' "
@@ -130,6 +142,7 @@ class OfflineImap:
parser.add_option("-k", dest="configoverride",
action="append",
+ type=str,
metavar="[section:]option=value",
help=
"""Override configuration file option. If"section" is
@@ -161,20 +174,28 @@ class OfflineImap:
"not usable. Possible interface choices are: %s " %
", ".join(UI_LIST.keys()))
+ parser.set_defaults(use_unicode=False)
(options, args) = parser.parse_args()
- globals.set_options (options)
+ globals.set_options(options)
+
+ if options.unicode_help:
+ uni.help_message()
+ sys.exit(0)
+ if options.use_unicode:
+ logging.info("Unicode support enabled")
#read in configuration file
+ # UNICODE: requires filesystem/console unicode support.
if not options.configfile:
# Try XDG location, then fall back to ~/.offlineimaprc
xdg_var = 'XDG_CONFIG_HOME'
if not xdg_var in os.environ or not os.environ[xdg_var]:
- xdg_home = os.path.expanduser('~/.config')
+ xdg_home = os.path.expanduser(u'~/.config')
else:
xdg_home = os.environ[xdg_var]
options.configfile = os.path.join(xdg_home, "offlineimap", "config")
if not os.path.exists(options.configfile):
- options.configfile = os.path.expanduser('~/.offlineimaprc')
+ options.configfile = os.path.expanduser(u'~/.offlineimaprc')
configfilename = options.configfile
else:
configfilename = os.path.expanduser(options.configfile)
@@ -182,10 +203,13 @@ class OfflineImap:
config = CustomConfigParser()
if not os.path.exists(configfilename):
# TODO, initialize and make use of chosen ui for logging
- logging.error(" *** Config file '%s' does not exist; aborting!"%
+ logging.error(u" *** Config file '%s' does not exist; aborting!"%
configfilename)
sys.exit(1)
- config.read(configfilename)
+ if options.use_unicode:
+ config.readfp(codecs.open(configfilename, 'rt', uni.ENCODING))
+ else:
+ config.read(configfilename)
#profile mode chosen?
if options.profiledir:
@@ -195,18 +219,21 @@ class OfflineImap:
options.singlethreading = True
if os.path.exists(options.profiledir):
# TODO, make use of chosen ui for logging
- logging.warn("Profile mode: Directory '%s' already exists!"%
+ logging.warn(u"Profile mode: Directory '%s' already exists!"%
options.profiledir)
else:
os.mkdir(options.profiledir)
threadutil.ExitNotifyThread.set_profiledir(options.profiledir)
# TODO, make use of chosen ui for logging
- logging.warn("Profile mode: Potentially large data will be "
+ logging.warn(u"Profile mode: Potentially large data will be "
"created in '%s'"% options.profiledir)
#override a config value
if options.configoverride:
+ if options.use_unicode:
+ options.configoverride = options.configoverride.decode(sys.stdin.encoding)
for option in options.configoverride:
+ # option is unicode
(key, value) = option.split('=', 1)
if ':' in key:
(secname, key) = key.split(':', 1)
@@ -219,13 +246,15 @@ class OfflineImap:
ui_type = config.getdefault('general', 'ui', 'ttyui')
if options.interface != None:
ui_type = options.interface
- if '.' in ui_type:
+ if options.use_unicode:
+ ui_type = uni.fs2uni(ui_type)
+ if u'.' in ui_type:
#transform Curses.Blinkenlights -> Blinkenlights
ui_type = ui_type.split('.')[-1]
# TODO, make use of chosen ui for logging
- logging.warning('Using old interface name, consider using one '
+ logging.warn(u'Using old interface name, consider using one '
'of %s'% ', '.join(UI_LIST.keys()))
- if options.diagnostics: ui_type = 'basic' # enforce basic UI for --info
+ if options.diagnostics: ui_type = u'basic' # enforce basic UI for --info
# dry-run? Set [general]dry-run=True
if options.dryrun:
@@ -236,8 +265,8 @@ class OfflineImap:
# create the ui class
self.ui = UI_LIST[ui_type.lower()](config)
except KeyError:
- logging.error("UI '%s' does not exist, choose one of: %s"% \
- (ui_type, ', '.join(UI_LIST.keys())))
+ logging.error(u"UI '%s' does not exist, choose one of: %s"%
+ (uni.uni2fs(ui_type), ', '.join(UI_LIST.keys())))
sys.exit(1)
setglobalui(self.ui)
@@ -250,10 +279,10 @@ class OfflineImap:
if options.debugtype:
self.ui.logger.setLevel(logging.DEBUG)
- if options.debugtype.lower() == 'all':
- options.debugtype = 'imap,maildir,thread'
+ if options.debugtype.lower() == u'all':
+ options.debugtype = u'imap,maildir,thread'
#force single threading?
- if not ('thread' in options.debugtype.split(',') \
+ if not (u'thread' in options.debugtype.split(',') \
and not options.singlethreading):
self.ui._msg("Debug mode: Forcing to singlethreaded.")
options.singlethreading = True
@@ -268,18 +297,23 @@ class OfflineImap:
if options.runonce:
# FIXME: spaghetti code alert!
for section in accounts.getaccountlist(config):
+ # section type is str
config.remove_option('Account ' + section, "autorefresh")
if options.quick:
for section in accounts.getaccountlist(config):
+ # section type is str
config.set('Account ' + section, "quick", '-1')
#custom folder list specified?
if options.folders:
+ if options.use_unicode:
+ options.folders = options.folders.decode(sys.stdin.encoding)
foldernames = options.folders.split(",")
folderfilter = "lambda f: f in %s"% foldernames
folderincludes = "[]"
for accountname in accounts.getaccountlist(config):
+ # accountname type is str
account_section = 'Account ' + accountname
remote_repo_section = 'Repository ' + \
config.get(account_section, 'remoterepository')
@@ -298,6 +332,7 @@ class OfflineImap:
config.getdefaultint('general', 'maxsyncaccounts', 1))
for reposname in config.getsectionlist('Repository'):
+ # reposname type is str
for instancename in ["FOLDER_" + reposname,
"MSGCOPY_" + reposname]:
if options.singlethreading:
@@ -314,6 +349,7 @@ class OfflineImap:
self.config is supposed to have been correctly initialized
already."""
+
try:
pidfd = open(self.config.getmetadatadir() + "/pid", "w")
pidfd.write(str(os.getpid()) + "\n")
@@ -325,6 +361,10 @@ class OfflineImap:
# Honor CLI --account option, only.
# Accounts to sync are put into syncaccounts variable.
activeaccounts = self.config.get("general", "accounts")
+ if options.use_unicode:
+ activeaccounts = uni.uni2str(activeaccounts, exception_msg=
+ "configuration: non ASCII character in account(s) '%s'"%
+ activeaccounts)
if options.accounts:
activeaccounts = options.accounts
activeaccounts = activeaccounts.replace(" ", "")
@@ -335,10 +375,10 @@ class OfflineImap:
for account in activeaccounts:
if account not in allaccounts:
if len(allaccounts) == 0:
- errormsg = "The account '%s' does not exist because no" \
+ errormsg = u"The account '%s' does not exist because no" \
" accounts are defined!"% account
else:
- errormsg = "The account '%s' does not exist. Valid ac" \
+ errormsg = u"The account '%s' does not exist. Valid ac" \
"counts are: %s"% \
(account, ", ".join(allaccounts.keys()))
self.ui.terminate(1, errormsg=errormsg)
@@ -351,11 +391,11 @@ class OfflineImap:
accounts.Account.set_abort_event(self.config, 1)
elif sig == signal.SIGUSR2:
# tell each account to stop looping
- getglobalui().warn("Terminating after this sync...")
+ getglobalui().warn(u"Terminating after this sync...")
accounts.Account.set_abort_event(self.config, 2)
elif sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
# tell each account to ABORT ASAP (ctrl-c)
- getglobalui().warn("Terminating NOW (this may "\
+ getglobalui().warn(u"Terminating NOW (this may "\
"take a few seconds)...")
accounts.Account.set_abort_event(self.config, 3)
elif sig == signal.SIGQUIT:
@@ -387,6 +427,11 @@ class OfflineImap:
except (SystemExit):
raise
except Exception as e:
+ # FIXME: UNICODE: This helps having good traces. There are some
+ # Exception with missing of full traces or other informations. This
+ # should be improved.
+ if globals.options.use_unicode:
+ raise
self.ui.error(e)
self.ui.terminate()
diff --git a/offlineimap/localeval.py b/offlineimap/localeval.py
index a9494fb..96601cf 100644
--- a/offlineimap/localeval.py
+++ b/offlineimap/localeval.py
@@ -17,6 +17,10 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import imp
+import codecs
+
+from offlineimap import globals
+from offlineimap.utils import uni
try:
import errno
except:
@@ -31,7 +35,11 @@ class LocalEval:
if path is not None:
# FIXME: limit opening files owned by current user with rights set
# to fixed mode 644.
- foo = open(path, 'r')
+ if globals.options.use_unicode:
+ path = uni.uni2fs(path)
+ foo = codecs.open(path, 'r', uni.ENCODING)
+ else:
+ foo = open(path, 'r')
module = imp.load_module(
'<none>',
foo,
@@ -41,6 +49,8 @@ class LocalEval:
self.namespace[attr] = getattr(module, attr)
def eval(self, text, namespace=None):
+ """eval() builtin use Latin-1 or UTF-8 arguments, also does this one."""
+
names = {}
names.update(self.namespace)
if namespace is not None:
diff --git a/offlineimap/mbnames.py b/offlineimap/mbnames.py
index 936a110..ed27c20 100644
--- a/offlineimap/mbnames.py
+++ b/offlineimap/mbnames.py
@@ -19,6 +19,9 @@
import os.path
import re # for folderfilter
from threading import Lock
+import codecs
+from offlineimap.utils import uni
+from offlineimap import globals
boxes = {}
localroots = {}
@@ -56,17 +59,25 @@ def __genmbnames():
localeval = config.getlocaleval()
if not config.getdefaultboolean("mbnames", "enabled", 0):
return
- path = config.apply_xforms(config.get("mbnames", "filename"), xforms)
- file = open(path, "wt")
- file.write(localeval.eval(config.get("mbnames", "header")))
+ path = config.get("mbnames", "filename")
+ path = config.apply_xforms(path, xforms)
+ if globals.options.use_unicode:
+ path = uni.uni2fs(path)
+ mbnames_file = codecs.open(path, "wt", uni.ENCODING)
+ mbnames_header = config.get("mbnames", "header")
+ mbnames_file.write(uni.uni2bytes(localeval.eval(mbnames_header)))
+ else:
+ mbnames_file = open(path, "wt")
+ mbnames_header = config.get("mbnames", "header")
+ mbnames_file.write(localeval.eval(mbnames_header))
folderfilter = lambda accountname, foldername: 1
if config.has_option("mbnames", "folderfilter"):
- folderfilter = localeval.eval(config.get("mbnames", "folderfilter"),
- {'re': re})
+ mbnames_folderfilter = config.get("mbnames", "folderfilter")
+ folderfilter = localeval.eval(mbnames_folderfilter, {'re': re})
mb_sort_keyfunc = lambda d: (d['accountname'], d['foldername'])
if config.has_option("mbnames", "sort_keyfunc"):
- mb_sort_keyfunc = localeval.eval(config.get("mbnames", "sort_keyfunc"),
- {'re': re})
+ mbnames_sort_keyfunc = config.get("mbnames", "sort_keyfunc")
+ mb_sort_keyfunc = localeval.eval(mbnames_sort_keyfunc, {'re': re})
itemlist = []
for accountname in boxes.keys():
localroot = localroots[accountname]
@@ -78,8 +89,15 @@ def __genmbnames():
itemlist.sort(key = mb_sort_keyfunc)
format_string = config.get("mbnames", "peritem", raw=1)
itemlist = [format_string % d for d in itemlist]
- file.write(localeval.eval(config.get("mbnames", "sep")).join(itemlist))
- file.write(localeval.eval(config.get("mbnames", "footer")))
- file.close()
+ mbnames_sep = config.get("mbnames", "sep")
+ mbnames_footer = config.get("mbnames", "footer")
+ eval_mbnames_sep = localeval.eval(mbnames_sep).join(itemlist)
+ eval_mbnames_footer = localeval.eval(mbnames_footer)
+ if globals.options.use_unicode:
+ eval_mbnames_sep = uni.uni2bytes(eval_mbnames_sep)
+ eval_mbnames_footer = uni.uni2bytes(eval_mbnames_footer)
+ mbnames_file.write(eval_mbnames_sep)
+ mbnames_file.write(eval_mbnames_footer)
+ mbnames_file.close()
finally:
mblock.release()
diff --git a/offlineimap/repository/Base.py b/offlineimap/repository/Base.py
index 0cf44f8..b35061f 100644
--- a/offlineimap/repository/Base.py
+++ b/offlineimap/repository/Base.py
@@ -19,8 +19,10 @@ import re
import os.path
from sys import exc_info
+from offlineimap import globals
from offlineimap import CustomConfig
from offlineimap.ui import getglobalui
+from offlineimap.utils import uni
from offlineimap.error import OfflineImapError
class BaseRepository(CustomConfig.ConfigHelperMixin, object):
@@ -29,7 +31,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
self.ui = getglobalui()
self.account = account
self.config = account.getconfig()
- self.name = reposname
+ self.name = reposname # Expected ASCII
self.localeval = account.getlocaleval()
self._accountname = self.account.getname()
self._readonly = self.getconfboolean('readonly', False)
@@ -44,9 +46,13 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
if not os.path.exists(self.uiddir):
os.mkdir(self.uiddir, 0o700)
+ # Expects Unicode syntaxt u''.
self.nametrans = lambda foldername: foldername
+ # Expects Unicode syntaxt u''.
self.folderfilter = lambda foldername: 1
+ # Expects Unicode syntaxt u''.
self.folderincludes = []
+ # Expects Unicode syntaxt u''.
self.foldersort = None
if self.config.has_option(self.getsection(), 'nametrans'):
self.nametrans = self.localeval.eval(
@@ -91,6 +97,9 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
return self.name
def __str__(self):
+ # Repository names are expected full ASCII but just in case...
+ if globals.options.use_unicode:
+ return uni.uni2fs(self.name)
return self.name
@property
@@ -196,7 +205,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
dst_haschanged = True # Need to refresh list
except OfflineImapError as e:
self.ui.error(e, exc_info()[2],
- "Creating folder %s on repository %s"%
+ u"Creating folder %s on repository %s"%
(src_name_t, dst_repo))
raise
status_repo.makefolder(src_name_t.replace(dst_repo.getsep(),
@@ -213,7 +222,7 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
# 1) would src repo filter out the new folder name? In this
# case don't create it on it:
if not self.should_sync_folder(dst_name_t):
- self.ui.debug('', "Not creating folder '%s' (repository '%s"
+ self.ui.debug('', u"Not creating folder '%s' (repository '%s"
"') as it would be filtered out on that repository."%
(dst_name_t, self))
continue
@@ -234,16 +243,15 @@ class BaseRepository(CustomConfig.ConfigHelperMixin, object):
"itories so they lead to identical names if applied bac"
"k and forth. 2) Use folderfilter settings on a reposit"
"ory to prevent some folders from being created on the "
- "other side." % (dst_folder.name, dst_repo, dst_name_t,
- src_repo, newdst_name),
- OfflineImapError.ERROR.REPO)
+ "other side."% (dst_folder.name, dst_repo, dst_name_t,
+ src_repo, newdst_name), OfflineImapError.ERROR.REPO)
# end sanity check, actually create the folder
try:
src_repo.makefolder(dst_name_t)
src_haschanged = True # Need to refresh list
except OfflineImapError as e:
- self.ui.error(e, exc_info()[2], "Creating folder %s on "
- "repository %s" % (dst_name_t, src_repo))
+ self.ui.error(e, exc_info()[2], u"Creating folder %s on "
+ "repository %s"% (dst_name_t, src_repo))
raise
status_repo.makefolder(dst_name_t.replace(
src_repo.getsep(), status_repo.getsep()))
diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py
index b109546..ff1e5fb 100644
--- a/offlineimap/repository/IMAP.py
+++ b/offlineimap/repository/IMAP.py
@@ -22,10 +22,11 @@ import netrc
import errno
from offlineimap.repository.Base import BaseRepository
-from offlineimap import folder, imaputil, imapserver, OfflineImapError
+from offlineimap import folder, imaputil, imapserver, OfflineImapError, globals
from offlineimap.folder.UIDMaps import MappedIMAPFolder
from offlineimap.threadutil import ExitNotifyThread
from offlineimap.utils.distro import get_os_sslcertfile, get_os_sslcertfile_searchpath
+from offlineimap.utils import uni
class IMAPRepository(BaseRepository):
@@ -138,10 +139,10 @@ class IMAPRepository(BaseRepository):
for m in mechs:
if m not in supported:
raise OfflineImapError("Repository %s: "% self + \
- "unknown authentication mechanism '%s'"% m,
- OfflineImapError.ERROR.REPO)
+ "unknown authentication mechanism '%s'"%
+ uni.uni2fs(m), OfflineImapError.ERROR.REPO)
- self.ui.debug('imap', "Using authentication mechanisms %s" % mechs)
+ self.ui.debug('imap', u"Using authentication mechanisms %s"% mechs)
return mechs
@@ -218,14 +219,14 @@ class IMAPRepository(BaseRepository):
xforms = [os.path.expanduser, os.path.expandvars, os.path.abspath]
cacertfile = self.getconf_xform('sslcacertfile', xforms, None)
- if self.getconf('sslcacertfile', None) == "OS-DEFAULT":
+ if self.getconf('sslcacertfile', None) == u"OS-DEFAULT":
cacertfile = get_os_sslcertfile()
if cacertfile == None:
searchpath = get_os_sslcertfile_searchpath()
if searchpath:
reason = "Default CA bundle was requested, "\
"but no existing locations available. "\
- "Tried %s." % (", ".join(searchpath))
+ "Tried %s."% (", ".join(searchpath))
else:
reason = "Default CA bundle was requested, "\
"but OfflineIMAP doesn't know any for your "\
@@ -235,7 +236,8 @@ class IMAPRepository(BaseRepository):
return None
if not os.path.isfile(cacertfile):
reason = "CA certfile for repository '%s' couldn't be found. "\
- "No such file: '%s'" % (self.name, cacertfile)
+ "No such file: '%s'"% \
+ (uni.uni2fs(self.name), uni.uni2fs(cacertfile))
raise OfflineImapError(reason, OfflineImapError.ERROR.REPO)
return cacertfile
@@ -297,7 +299,10 @@ class IMAPRepository(BaseRepository):
# 3. read password from file specified in Repository 'remotepassfile'
passfile = self.getconf('remotepassfile', None)
if passfile != None:
- fd = open(os.path.expanduser(passfile))
+ f = os.path.expanduser(passfile)
+ if globals.options.use_unicode:
+ f = uni.uni2fs(f)
+ fd = open(f)
password = fd.readline().strip()
fd.close()
return password
@@ -353,7 +358,10 @@ class IMAPRepository(BaseRepository):
if self.getconfboolean('subscribedonly', False):
listfunction = imapobj.lsub
try:
- listresult = listfunction(directory = self.imapserver.reference)[1]
+ ref = self.imapserver.reference
+ if globals.options.use_unicode:
+ ref = uni.imap2uni(ref)
+ listresult = listfunction(directory=ref)[1]
finally:
self.imapserver.releaseconnection(imapobj)
for s in listresult:
@@ -367,6 +375,8 @@ class IMAPRepository(BaseRepository):
if '\\noselect' in flaglist:
continue
foldername = imaputil.dequote(name)
+ if globals.options.use_unicode:
+ foldername = uni.imap2uni(foldername)
retval.append(self.getfoldertype()(self.imapserver, foldername,
self))
# Add all folderincludes
@@ -375,7 +385,11 @@ class IMAPRepository(BaseRepository):
try:
for foldername in self.folderincludes:
try:
- imapobj.select(foldername, readonly = True)
+ if globals.options.use_unicode:
+ imap_foldername = uni.uni2imap(foldername)
+ else:
+ imap_foldername = foldername
+ imapobj.select(imap_foldername, readonly = True)
except OfflineImapError as e:
# couldn't select this folderinclude, so ignore folder.
if e.severity > OfflineImapError.ERROR.FOLDER:
@@ -390,7 +404,7 @@ class IMAPRepository(BaseRepository):
if self.foldersort is None:
# default sorting by case insensitive transposed name
- retval.sort(key=lambda x: str.lower(x.getvisiblename()))
+ retval.sort(key=lambda x: x.getvisiblename().lower())
else:
# do foldersort in a python3-compatible way
# http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function
@@ -426,11 +440,13 @@ class IMAPRepository(BaseRepository):
return
imapobj = self.imapserver.acquireconnection()
try:
+ if globals.options.use_unicode:
+ foldername = uni.uni2imap(foldername)
result = imapobj.create(foldername)
if result[0] != 'OK':
raise OfflineImapError("Folder '%s'[%s] could not be created. "
- "Server responded: %s"% (foldername, self, str(result)),
- OfflineImapError.ERROR.FOLDER)
+ "Server responded: %s"% (uni.uni2fs(foldername), self,
+ str(result)), OfflineImapError.ERROR.FOLDER)
finally:
self.imapserver.releaseconnection(imapobj)
diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py
index fc34a55..a6e0595 100644
--- a/offlineimap/repository/LocalStatus.py
+++ b/offlineimap/repository/LocalStatus.py
@@ -20,6 +20,7 @@ import os
from offlineimap.folder.LocalStatus import LocalStatusFolder
from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder
from offlineimap.repository.Base import BaseRepository
+from offlineimap.utils import uni
class LocalStatusRepository(BaseRepository):
def __init__(self, reposname, account):
@@ -54,7 +55,7 @@ class LocalStatusRepository(BaseRepository):
else:
raise SyntaxWarning("Unknown status_backend '%s' for account '%s'"%
- (backend, self.account.name))
+ (uni.uni2fs(backend), uni.uni2fs(self.account.name)))
def import_other_backend(self, folder):
for bk, dic in self.backends.items():
@@ -68,7 +69,7 @@ class LocalStatusRepository(BaseRepository):
# if backend contains data, import it to folder.
if not folderbk.isnewfolder():
- self.ui._msg('Migrating LocalStatus cache from %s to %s " \
+ self.ui._msg(u'Migrating LocalStatus cache from %s to %s " \
"status folder for %s:%s'%
(bk, self._backend, self.name, folder.name))
@@ -88,7 +89,22 @@ class LocalStatusRepository(BaseRepository):
# Create an empty StatusFolder
folder = self.LocalStatusFolderClass(foldername, self)
- folder.save()
+ # With Unicode, makefolder() might be raised while cache file already
+ # exists. Calling makefolder() let us update the folder name on disk in
+ # a Maildir if the folder name is not full ASCII.
+ #
+ # Avoid clearing the folder cache file if it already exists. We don't
+ # want to re-download emails from that folder if we already have a cache
+ # file.
+ #
+ # Make the check here to not annoy callers with this issue and because
+ # we can't insert it in the save() method simply (missing of context).
+ #
+ # FIXME: why this call to save() in the first place? We have NOTHING to
+ # save at all. This will only create the cache file which case should be
+ # handled from within the class...
+ if folder.isnewfolder():
+ folder.save()
# Invalidate the cache.
self.forgetfolders()
diff --git a/offlineimap/repository/Maildir.py b/offlineimap/repository/Maildir.py
index 0262ba2..a5b26b0 100644
--- a/offlineimap/repository/Maildir.py
+++ b/offlineimap/repository/Maildir.py
@@ -15,12 +15,14 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-from offlineimap import folder
+import os
+from stat import *
+
+from offlineimap import folder, globals
from offlineimap.ui import getglobalui
+from offlineimap.utils import uni
from offlineimap.error import OfflineImapError
from offlineimap.repository.Base import BaseRepository
-import os
-from stat import *
class MaildirRepository(BaseRepository):
def __init__(self, reposname, account):
@@ -32,7 +34,7 @@ class MaildirRepository(BaseRepository):
self.root = self.getlocalroot()
self.folders = None
self.ui = getglobalui()
- self.debug("MaildirRepository initialized, sep is %s"% repr(self.getsep()))
+ self.debug(u"MaildirRepository initialized, sep is %s"% repr(self.getsep()))
self.folder_atimes = []
# Create the top-level folder if it doesn't exist
@@ -42,7 +44,10 @@ class MaildirRepository(BaseRepository):
def _append_folder_atimes(self, foldername):
"""Store the atimes of a folder's new|cur in self.folder_atimes"""
- p = os.path.join(self.root, foldername)
+ if globals.options.use_unicode:
+ fs_root = uni.uni2fs(self.root)
+ foldername = uni.uni2fs(foldername)
+ p = os.path.join(fs_root, foldername)
new = os.path.join(p, 'new')
cur = os.path.join(p, 'cur')
atimes = (p, os.path.getatime(new), os.path.getatime(cur))
@@ -67,7 +72,7 @@ class MaildirRepository(BaseRepository):
return self.getconf_xform('localfolders', xforms)
def debug(self, msg):
- self.ui.debug('maildir', msg)
+ self.ui.debug(u'maildir', msg)
def getsep(self):
return self.getconf('sep', '.').strip()
@@ -96,17 +101,25 @@ class MaildirRepository(BaseRepository):
assert not component in ['new', 'cur', 'tmp'],\
"When using nested folders (/ as a Maildir separator), "\
"folder names may not contain 'new', 'cur', 'tmp'."
- assert foldername.find('../') == -1, "Folder names may not contain ../"
- assert not foldername.startswith('/'), "Folder names may not begin with /"
+ assert foldername.find(u'../') == -1, "Folder names may not contain ../"
+ assert not foldername.startswith(u'/'), "Folder names may not begin with /"
+
+ diverged, unexpected, expected = uni.diverged_foldernames(foldername,
+ globals.options.use_unicode)
+ if diverged:
+ fs_root = uni.uni2fs(self.root)
+ uni.rename_diverged(fs_root, unexpected, expected)
+ self.debug(" makefolder : renamed '%s' to '%s'"%
+ (uni.bytes2uni(unexpected), uni.bytes2uni(expected)))
# If we're using hierarchical folders, it's possible that
# sub-folders may be created before higher-up ones.
- self.debug("makefolder: calling makedirs '%s'"% full_path)
+ self.debug(u"makefolder: calling makedirs '%s'"% full_path)
try:
os.makedirs(full_path, 0o700)
except OSError as e:
if e.errno == 17 and os.path.isdir(full_path):
- self.debug("makefolder: '%s' already a directory"% foldername)
+ self.debug(u"makefolder: '%s' already a directory"% foldername)
else:
raise
for subdir in ['cur', 'new', 'tmp']:
@@ -114,13 +127,13 @@ class MaildirRepository(BaseRepository):
os.mkdir(os.path.join(full_path, subdir), 0o700)
except OSError as e:
if e.errno == 17 and os.path.isdir(full_path):
- self.debug("makefolder: '%s' already has subdir %s"%
+ self.debug(u"makefolder: '%s' already has subdir %s"%
(foldername, subdir))
else:
raise
def deletefolder(self, foldername):
- self.ui.warn("NOT YET IMPLEMENTED: DELETE FOLDER %s"% foldername)
+ self.ui.warn(u"NOT YET IMPLEMENTED: DELETE FOLDER %s"% foldername)
def getfolder(self, foldername):
"""Return a Folder instance of this Maildir
@@ -135,8 +148,8 @@ class MaildirRepository(BaseRepository):
if foldername == f.name:
return f
raise OfflineImapError("getfolder() asked for a nonexisting "
- "folder '%s'."% foldername,
- OfflineImapError.ERROR.FOLDER)
+ "folder '%s'."% uni.uni2fs(foldername),
+ OfflineImapError.ERROR.FOLDER)
def _getfolders_scandir(self, root, extension=None):
"""Recursively scan folder 'root'; return a list of MailDirFolder
@@ -144,8 +157,8 @@ class MaildirRepository(BaseRepository):
:param root: (absolute) path to Maildir root
:param extension: (relative) subfolder to examine within root"""
- self.debug("_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s"%
- (root, extension))
+ self.debug(u"_GETFOLDERS_SCANDIR STARTING. root = %s, extension = %s"%
+ (uni.uni2std(root), uni.uni2std(extension)))
retval = []
# Configure the full path to this repository -- "toppath"
@@ -153,21 +166,66 @@ class MaildirRepository(BaseRepository):
toppath = os.path.join(root, extension)
else:
toppath = root
- self.debug(" toppath = %s"% toppath)
+ self.debug(u" toppath = %s (type: %s)"% (uni.uni2std(toppath),
+ str(type(toppath))))
# Iterate over directories in top & top itself.
for dirname in os.listdir(toppath) + ['']:
- self.debug(" dirname = %s"% dirname)
+ # If we run without Unicode support, prevent from non full ASCII
+ # dirname. In legacy mode, folders are encoded to IMAP UTF-7 (full
+ # ASCII).
+
if dirname == '' and extension is not None:
- self.debug(' skip this entry (already scanned)')
+ self.debug(u' skip this entry (already scanned)')
continue
+
+ if dirname != '':
+ diverged, unexpected, expected = uni.diverged_foldernames(dirname,
+ globals.options.use_unicode)
+ if diverged:
+ if globals.options.use_unicode:
+ dirname = uni.imap2uni(dirname)
+ fs_toppath = uni.uni2fs(toppath)
+ else:
+ fs_toppath = toppath
+ uni.rename_diverged(fs_toppath, unexpected, expected)
+ self.debug(" scandir folders: renamed '%s' to '%s'"%
+ (uni.bytes2uni(unexpected), uni.bytes2uni(expected)))
+ # Sanity check.
+ if not globals.options.use_unicode and not uni.isASCII(dirname):
+ self.debug(u" skip dirname (unexpected character) = %s"%
+ uni.uni2std(dirname))
+ continue
+
+ self.debug(u" dirname = %s"% dirname)
if dirname in ['cur', 'new', 'tmp']:
self.debug(" skip this entry (Maildir special)")
# Bypass special files.
continue
+
+ if dirname != '':
+ diverged, unexpected, expected = uni.diverged_foldernames(dirname,
+ globals.options.use_unicode)
+ if diverged:
+ if globals.options.use_unicode:
+ dirname = uni.imap2uni(dirname)
+ fs_toppath = uni.uni2fs(toppath)
+ else:
+ fs_toppath = toppath
+ uni.rename_diverged(fs_toppath, unexpected, expected)
+ self.debug(" scandir folders: renamed '%s' to '%s'"%
+ (uni.bytes2uni(unexpected), uni.bytes2uni(expected)))
+ # Sanity check.
+ if not globals.options.use_unicode and not uni.isASCII(dirname):
+ self.debug(u" skip dirname (unexpected character) = %s"% dirname)
+ continue
+
+ self.debug(u" dirname = %s"% dirname)
fullname = os.path.join(toppath, dirname)
+ if globals.options.use_unicode:
+ fullname = uni.uni2fs(fullname)
if not os.path.isdir(fullname):
- self.debug(" skip this entry (not a directory)")
+ self.debug(u" skip this entry (not a directory)")
# Not a directory -- not a folder.
continue
# extension can be None.
@@ -180,7 +238,7 @@ class MaildirRepository(BaseRepository):
os.path.isdir(os.path.join(fullname, 'new')) and
os.path.isdir(os.path.join(fullname, 'tmp'))):
# This directory has maildir stuff -- process
- self.debug(" This is maildir folder '%s'."% foldername)
+ self.debug(u" This is maildir folder '%s'."% foldername)
if self.getconfboolean('restoreatime', False):
self._append_folder_atimes(foldername)
fd = self.getfoldertype()(self.root, foldername,
@@ -190,7 +248,7 @@ class MaildirRepository(BaseRepository):
if self.getsep() == '/' and dirname != '':
# Recursively check sub-directories for folders too.
retval.extend(self._getfolders_scandir(root, foldername))
- self.debug("_GETFOLDERS_SCANDIR RETURNING %s"% \
+ self.debug(u"_GETFOLDERS_SCANDIR RETURNING %s"% \
repr([x.getname() for x in retval]))
return retval
diff --git a/offlineimap/repository/__init__.py b/offlineimap/repository/__init__.py
index 0fbbc13..e609586 100644
--- a/offlineimap/repository/__init__.py
+++ b/offlineimap/repository/__init__.py
@@ -28,6 +28,7 @@ from offlineimap.repository.Maildir import MaildirRepository
from offlineimap.repository.GmailMaildir import GmailMaildirRepository
from offlineimap.repository.LocalStatus import LocalStatusRepository
from offlineimap.error import OfflineImapError
+from offlineimap.utils import uni
class Repository(object):
@@ -58,7 +59,8 @@ class Repository(object):
return LocalStatusRepository(name, account)
else:
- errstr = "Repository type %s not supported" % reqtype
+ errstr = "Repository type %s not supported"% \
+ uni.uni2fs(reqtype)
raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO)
# Get repository type.
@@ -67,7 +69,8 @@ class Repository(object):
repostype = config.get('Repository ' + name, 'type').strip()
except NoSectionError as e:
errstr = ("Could not find section '%s' in configuration. Required "
- "for account '%s'." % ('Repository %s' % name, account))
+ "for account '%s'."%
+ ('Repository %s'% uni.uni2fs(name), uni.uni2fs(account)))
raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \
None, exc_info()[2]
@@ -75,7 +78,7 @@ class Repository(object):
repo = typemap[repostype]
except KeyError:
errstr = "'%s' repository not supported for '%s' repositories."% \
- (repostype, reqtype)
+ (uni.uni2fs(repostype), uni.uni2fs(reqtype))
raise OfflineImapError(errstr, OfflineImapError.ERROR.REPO), \
None, exc_info()[2]
diff --git a/offlineimap/threadutil.py b/offlineimap/threadutil.py
index f69f8a6..04c3890 100644
--- a/offlineimap/threadutil.py
+++ b/offlineimap/threadutil.py
@@ -23,7 +23,10 @@ except ImportError: # python3
import traceback
import os.path
import sys
+
from offlineimap.ui import getglobalui
+from offlineimap.utils import uni
+
######################################################################
# General utilities
@@ -172,7 +175,8 @@ class ExitNotifyThread(Thread):
except SystemExit:
pass
prof.dump_stats(os.path.join(ExitNotifyThread.profiledir,
- "%s_%s.prof"% (self.ident, self.getName())))
+ "%s_%s.prof"% (uni.uni2fs(self.ident),
+ uni.uni2fs(self.getName()))))
except Exception as e:
# Thread exited with Exception, store it
tb = traceback.format_exc()
diff --git a/offlineimap/ui/Curses.py b/offlineimap/ui/Curses.py
index ddc05ea..cbf68ee 100644
--- a/offlineimap/ui/Curses.py
+++ b/offlineimap/ui/Curses.py
@@ -25,6 +25,8 @@ import logging
from offlineimap.ui.UIBase import UIBase
from offlineimap.threadutil import ExitNotifyThread
+from offlineimap.utils import uni
+from offlineimap import globals
import offlineimap
@@ -180,7 +182,8 @@ class CursesAccountFrame:
# if this belongs to an Account (and not *Control), set the
# skipsleep pref
if isinstance(self.account, offlineimap.accounts.Account):
- self.ui.info("Requested synchronization for acc: %s"% self.account)
+ self.ui.info(u"Requested synchronization for acc: %s"%
+ self.account)
self.account.config.set('Account %s'% self.account.name,
'skipsleep', '1')
@@ -301,6 +304,8 @@ class CursesLogHandler(logging.StreamHandler):
"""self.ui has been set to the UI class before anything is invoked"""
def emit(self, record):
+ if globals.options.use_unicode:
+ record = uni.uni2std(record)
log_str = logging.StreamHandler.format(self, record)
color = self.ui.gettf().curses_color
# We must acquire both locks. Otherwise, deadlock can result.
@@ -348,7 +353,12 @@ class Blinkenlights(UIBase, CursesUtil):
ch = CursesLogHandler()
#ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
- self.formatter = logging.Formatter("%(message)s")
+ if globals.options.use_unicode:
+ encode_function = uni.uni2fs
+ else:
+ encode_function = None
+ self.formatter = uni.UnicodeFormatter("%(message)s",
+ encode_function=encode_function)
ch.setFormatter(self.formatter)
# add the handlers to the logger
self.logger.addHandler(ch)
@@ -511,7 +521,7 @@ class Blinkenlights(UIBase, CursesUtil):
return
if chr(key) == 'q':
# Request to quit completely.
- self.warn("Requested shutdown via 'q'")
+ self.warn(u"Requested shutdown via 'q'")
offlineimap.accounts.Account.set_abort_event(self.config, 3)
try:
index = int(chr(key))
@@ -525,7 +535,7 @@ class Blinkenlights(UIBase, CursesUtil):
def sleep(self, sleepsecs, account):
self.gettf().setcolor('red')
- self.info("Next sync in %d:%02d"% (sleepsecs / 60, sleepsecs % 60))
+ self.info(u"Next sync in %d:%02d"% (sleepsecs / 60, sleepsecs % 60))
return super(Blinkenlights, self).sleep(sleepsecs, account)
def sleeping(self, sleepsecs, remainingsecs):
@@ -551,9 +561,9 @@ class Blinkenlights(UIBase, CursesUtil):
self.lock()
try:
#s.gettf().setcolor('white')
- self.warn(" *** Input Required")
- self.warn(" *** Please enter password for account %s: " % \
- accountname)
+ self.warn(u" *** Input Required")
+ self.warn(u" *** Please enter password for account %s: "%
+ accountname)
self.logwin.refresh()
password = self.logwin.getstr()
finally:
diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py
index dc650c3..feac5d3 100644
--- a/offlineimap/ui/Machine.py
+++ b/offlineimap/ui/Machine.py
@@ -20,14 +20,29 @@ except ImportError: # python3
import sys
import time
import logging
+
+from offlineimap import globals
from threading import currentThread
from offlineimap.ui.UIBase import UIBase
+from offlineimap.utils import uni
import offlineimap
protocol = '7.0.0'
-class MachineLogFormatter(logging.Formatter):
+
+def getThreadname(thread):
+ thr_name = thread.getName()
+ if globals.options.use_unicode:
+ thr_name = uni.bytes2uni(thr_name)
+
+
+class MachineLogFormatter(uni.UnicodeFormatter):
"""urlencodes any outputted line, to avoid multi-line output"""
+
+ def __init__(self, fmt, datefmt=None):
+ uni.UnicodeFormatter.__init__(self, fmt, datefmt,
+ encode_function=uni.uni2bytes)
+
def format(s, record):
# Mapping of log levels to historic tag names
severity_map = {
@@ -43,7 +58,7 @@ class MachineLogFormatter(logging.Formatter):
whoami = record.machineui["id"]
else:
command = ""
- whoami = currentThread().getName()
+ whoami = getThreadname(currentThread())
prefix = "%s:%s"% (command, urlencode([('', whoami)])[1:])
return "%s:%s:%s"% (severity, prefix, urlencode([('', line)])[1:])
@@ -67,7 +82,7 @@ class MachineUI(UIBase):
extra = {
'machineui': {
'command': command,
- 'id': currentThread().getName(),
+ 'id': getThreadname(currentThread())
}
})
@@ -84,7 +99,7 @@ class MachineUI(UIBase):
def unregisterthread(s, thread):
UIBase.unregisterthread(s, thread)
- s._printData(s.logger.info, 'unregisterthread', thread.getName())
+ s._printData(s.logger.info, 'unregisterthread', getThreadname(thread))
def debugging(s, debugtype):
s._printData(s.logger.debug, 'debugging', debugtype)
@@ -152,7 +167,7 @@ class MachineUI(UIBase):
def threadException(s, thread):
s._printData(s.logger.warning, 'threadException', "%s\n%s"%
- (thread.getName(), s.getThreadExceptionString(thread)))
+ (getThreadname(thread), s.getThreadExceptionString(thread)))
s.delThreadDebugLog(thread)
s.terminate(100)
@@ -164,7 +179,7 @@ class MachineUI(UIBase):
s._printData(s.logger.warning, 'mainException', s.getMainExceptionString())
def threadExited(s, thread):
- s._printData(s.logger.info, 'threadExited', thread.getName())
+ s._printData(s.logger.info, 'threadExited', getThreadname(thread))
UIBase.threadExited(s, thread)
def sleeping(s, sleepsecs, remainingsecs):
diff --git a/offlineimap/ui/TTY.py b/offlineimap/ui/TTY.py
index 0b5aa6a..b1203f2 100644
--- a/offlineimap/ui/TTY.py
+++ b/offlineimap/ui/TTY.py
@@ -19,31 +19,45 @@ import logging
import sys
import time
from getpass import getpass
+
+from offlineimap import globals
from offlineimap import banner
from offlineimap.ui.UIBase import UIBase
+from offlineimap.utils import uni
+
-class TTYFormatter(logging.Formatter):
+class TTYFormatter(uni.UnicodeFormatter):
"""Specific Formatter that adds thread information to the log output."""
- def __init__(self, *args, **kwargs):
+ def __init__(self, fmt, datefmt=None):
#super() doesn't work in py2.6 as 'logging' uses old-style class
- logging.Formatter.__init__(self, *args, **kwargs)
+ if globals.options.use_unicode:
+ encode_function = uni.uni2fs
+ else:
+ encode_function = None
+ uni.UnicodeFormatter.__init__(self, fmt, datefmt,
+ encode_function=encode_function)
self._last_log_thread = None
def format(self, record):
- """Override format to add thread information."""
+ """Override format to add thread information.
- #super() doesn't work in py2.6 as 'logging' uses old-style class
- log_str = logging.Formatter.format(self, record)
+ We are near to output. It's convenient to encode everything here."""
+
+ # super() doesn't work in py2.6 as 'logging' uses old-style class.
+ log_str = uni.UnicodeFormatter.format(self, record)
# If msg comes from a different thread than our last, prepend
# thread info. Most look like 'Account sync foo' or 'Folder
# sync foo'.
- t_name = record.threadName
+ #
+ # Because the Thread module does not support Unicode, threadName is
+ # encoded with uni.ENCODING.
+ t_name = uni.uni2fs(uni.bytes2uni(record.threadName))
if t_name == 'MainThread':
- return log_str # main thread doesn't get things prepended
+ return log_str # Main thread doesn't get things prepended.
if t_name != self._last_log_thread:
self._last_log_thread = t_name
- log_str = "%s:\n %s" % (t_name, log_str)
+ log_str = "%s:\n %s"% (t_name, log_str)
else:
log_str = " %s"% log_str
return log_str
@@ -78,7 +92,10 @@ class TTYUI(UIBase):
"""TTYUI backend is capable of querying the password."""
if errmsg:
- self.warn("%s: %s"% (accountname, errmsg))
+ msg = u"%s: %s"% (accountname, errmsg)
+ if globals.options.use_unicode:
+ msg = uni.uni2bytes(msg)
+ self.warn(msg)
self._log_con_handler.acquire() # lock the console output
try:
return getpass("Enter password for account '%s': " % accountname)
@@ -105,7 +122,7 @@ class TTYUI(UIBase):
if sleepsecs > 0:
if remainingsecs//60 != (remainingsecs-sleepsecs)//60:
- self.logger.info("Next refresh in %.1f minutes" % (
- remainingsecs/60.0))
+ self.logger.info(u"Next refresh in %.1f minutes"%
+ (remainingsecs/60.0))
time.sleep(sleepsecs)
return 0
diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py
index 9285b51..a06d8e5 100644
--- a/offlineimap/ui/UIBase.py
+++ b/offlineimap/ui/UIBase.py
@@ -26,6 +26,9 @@ try:
except ImportError: #python3
from queue import Queue
from collections import deque
+
+from offlineimap import globals
+from offlineimap.utils import uni
from offlineimap.error import OfflineImapError
import offlineimap
@@ -82,7 +85,12 @@ class UIBase(object):
ch = logging.StreamHandler(sys.stdout)
#ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
- self.formatter = logging.Formatter("%(message)s")
+ if globals.options.use_unicode:
+ encode_function = uni.uni2fs
+ else:
+ encode_function = None
+ self.formatter = uni.UnicodeFormatter("%(message)s",
+ encode_function=encode_function)
ch.setFormatter(self.formatter)
# add the handlers to the logger
self.logger.addHandler(ch)
@@ -92,16 +100,22 @@ class UIBase(object):
def setlogfile(self, logfile):
"""Create file handler which logs to file."""
- fh = logging.FileHandler(logfile, 'at')
- file_formatter = logging.Formatter("%(asctime)s %(levelname)s: "
- "%(message)s", '%Y-%m-%d %H:%M:%S')
+ if globals.options.use_unicode:
+ encode_function = uni.uni2fs
+ # logfile was kept encoded.
+ else:
+ encoding = 'ascii'
+ fh = logging.FileHandler(logfile, 'at', encoding=encoding)
+ file_formatter = uni.UnicodeFormatter("%(asctime)s %(levelname)s: "
+ "%(message)s", '%Y-%m-%d %H:%M:%S',
+ encode_function=None)
fh.setFormatter(file_formatter)
self.logger.addHandler(fh)
# write out more verbose initial info blurb on the log file
p_ver = ".".join([str(x) for x in sys.version_info[0:3]])
msg = "OfflineImap %s starting...\n Python: %s Platform: %s\n "\
- "Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform,
- " ".join(sys.argv))
+ u"Args: %s"% (offlineimap.__bigversion__, p_ver, sys.platform,
+ u" ".join(sys.argv))
self.logger.info(msg)
def _msg(self, msg):
@@ -142,10 +156,11 @@ class UIBase(object):
ui.error(exc, sys.exc_info()[2], msg="While syncing Folder %s in "
"repo %s")
"""
+
if msg:
- self.logger.error("ERROR: %s\n %s" % (msg, exc))
+ self.logger.error(u"ERROR: %s\n %s"% (msg, exc))
else:
- self.logger.error("ERROR: %s" % (exc))
+ self.logger.error(u"ERROR: %s"% (exc))
instant_traceback = exc_traceback
if not self.debuglist:
@@ -160,14 +175,17 @@ class UIBase(object):
"""Register current thread as being associated with an account name."""
cur_thread = threading.currentThread()
+ thr_name = cur_thread.getName()
+ if globals.options.use_unicode:
+ thr_name = uni.bytes2uni(thr_name)
if cur_thread in self.threadaccounts:
# was already associated with an old account, update info
- self.debug('thread', "Register thread '%s' (previously '%s', now "
- "'%s')" % (cur_thread.getName(),
- self.getthreadaccount(cur_thread), account))
+ self.debug('thread', u"Register thread '%s' "
+ "(previously '%s', now '%s')"% (thr_name,
+ self.getthreadaccount(cur_thread), account))
else:
- self.debug('thread', "Register new thread '%s' (account '%s')"%
- (cur_thread.getName(), account))
+ self.debug('thread', u"Register new thread '%s' (account '%s')"%
+ (thr_name, account))
self.threadaccounts[cur_thread] = account
def unregisterthread(self, thr):
@@ -175,7 +193,10 @@ class UIBase(object):
if thr in self.threadaccounts:
del self.threadaccounts[thr]
- self.debug('thread', "Unregister thread '%s'" % thr.getName())
+ thr_name = thr.getName()
+ if globals.options.use_unicode:
+ thr_name = uni.bytes2uni(thr_name)
+ self.debug('thread', u"Unregister thread '%s'"% thr_name)
def getthreadaccount(self, thr=None):
"""Get Account() for a thread (current if None)
@@ -195,14 +216,14 @@ class UIBase(object):
# introduced in p2.6 only, so we'll need to work around and
# shorten our debugmsg list manually :-(
self.debugmessages[cur_thread] = deque()
- self.debugmessages[cur_thread].append("%s: %s" % (debugtype, msg))
+ self.debugmessages[cur_thread].append(u"%s: %s"% (debugtype, msg))
# Shorten queue if needed
if len(self.debugmessages[cur_thread]) > self.debugmsglen:
self.debugmessages[cur_thread].popleft()
if debugtype in self.debuglist: # log if we are supposed to do so
- self.logger.debug("[%s]: %s" % (debugtype, msg))
+ self.logger.debug("u[%s]: %s"% (debugtype, msg))
def add_debug(self, debugtype):
global debugtypes
@@ -215,11 +236,11 @@ class UIBase(object):
def debugging(self, debugtype):
global debugtypes
- self.logger.debug("Now debugging for %s: %s" % (debugtype,
- debugtypes[debugtype]))
+ msg = u"Now debugging for %s: %s"% (debugtype, debugtypes[debugtype])
+ self.logger.debug(msg)
def invaliddebug(self, debugtype):
- self.warn("Invalid debug type: %s" % debugtype)
+ self.warn(u"Invalid debug type: %s"% debugtype)
def getnicename(self, object):
"""Return the type of a repository or Folder as string.
@@ -244,7 +265,7 @@ class UIBase(object):
" in this UI backend.")
def folderlist(self, folder_list):
- return ', '.join(["%s[%s]"% \
+ return u', '.join(["%s[%s]"% \
(self.getnicename(x), x.getname()) for x in folder_list])
################################################## WARNINGS
@@ -252,7 +273,7 @@ class UIBase(object):
if self.config.has_option('general', 'ignore-readonly') and \
self.config.getboolean('general', 'ignore-readonly'):
return
- self.warn("Attempted to synchronize message %d to folder %s[%s], "
+ self.warn(u"Attempted to synchronize message %d to folder %s[%s], "
"but that folder is read-only. The message will not be "
"copied to that folder."% (
uid, self.getnicename(destfolder), destfolder))
@@ -261,7 +282,7 @@ class UIBase(object):
if self.config.has_option('general', 'ignore-readonly') and \
self.config.getboolean('general', 'ignore-readonly'):
return
- self.warn("Attempted to modify flags for messages %s in folder %s[%s], "
+ self.warn(u"Attempted to modify flags for messages %s in folder %s[%s], "
"but that folder is read-only. No flags have been modified "
"for that message."% (
str(uidlist), self.getnicename(destfolder), destfolder))
@@ -270,7 +291,7 @@ class UIBase(object):
if self.config.has_option('general', 'ignore-readonly') and \
self.config.getboolean('general', 'ignore-readonly'):
return
- self.warn("Attempted to modify labels for messages %s in folder %s[%s], "
+ self.warn(u"Attempted to modify labels for messages %s in folder %s[%s], "
"but that folder is read-only. No labels have been modified "
"for that message."% (
str(uidlist), self.getnicename(destfolder), destfolder))
@@ -279,7 +300,7 @@ class UIBase(object):
if self.config.has_option('general', 'ignore-readonly') and \
self.config.getboolean('general', 'ignore-readonly'):
return
- self.warn("Attempted to delete messages %s in folder %s[%s], but that "
+ self.warn(u"Attempted to delete messages %s in folder %s[%s], but that "
"folder is read-only. No messages have been deleted in that "
"folder."% (str(uidlist), self.getnicename(destfolder),
destfolder))
@@ -301,159 +322,170 @@ class UIBase(object):
hostname = hostname if hostname else ''
port = "%s"% port if port else ''
if hostname:
- displaystr = ' to %s:%s' % (hostname, port)
- self.logger.info("Establishing connection%s" % displaystr)
+ displaystr = ' to %s:%s'% (hostname, port)
+ self.logger.info(u"Establishing connection%s"% displaystr)
def acct(self, account):
"""Output that we start syncing an account (and start counting)."""
self.acct_startimes[account] = time.time()
- self.logger.info("*** Processing account %s" % account)
+ self.logger.info(u"*** Processing account %s"% account)
def acctdone(self, account):
"""Output that we finished syncing an account (in which time)."""
sec = time.time() - self.acct_startimes[account]
del self.acct_startimes[account]
- self.logger.info("*** Finished account '%s' in %d:%02d"%
+ self.logger.info(u"*** Finished account '%s' in %d:%02d"%
(account, sec // 60, sec % 60))
def syncfolders(self, src_repo, dst_repo):
"""Log 'Copying folder structure...'."""
if self.logger.isEnabledFor(logging.DEBUG):
- self.debug('', "Copying folder structure from %s to %s" %\
- (src_repo, dst_repo))
+ self.debug('', u"Copying folder structure from %s to %s"%
+ (src_repo, dst_repo))
############################## Folder syncing
def makefolder(self, repo, foldername):
"""Called when a folder is created."""
- prefix = "[DRYRUN] " if self.dryrun else ""
- self.info(("{0}Creating folder {1}[{2}]".format(
+ prefix = u"[DRYRUN] " if self.dryrun else u""
+ self.info((u"{0}Creating folder {1}[{2}]".format(
prefix, foldername, repo)))
def syncingfolder(self, srcrepos, srcfolder, destrepos, destfolder):
"""Called when a folder sync operation is started."""
- self.logger.info("Syncing %s: %s -> %s"% (srcfolder,
+ self.logger.info(u"Syncing %s: %s -> %s"% (srcfolder,
self.getnicename(srcrepos),
self.getnicename(destrepos)))
def skippingfolder(self, folder):
"""Called when a folder sync operation is started."""
- self.logger.info("Skipping %s (not changed)" % folder)
+
+ self.logger.info(u"Skipping %s (not changed)"% folder)
def validityproblem(self, folder):
- self.logger.warning("UID validity problem for folder %s (repo %s) "
- "(saved %d; got %d); skipping it. Please see FAQ "
- "and manual on how to handle this."% \
- (folder, folder.getrepository(),
- folder.get_saveduidvalidity(), folder.get_uidvalidity()))
+ msg = u"UID validity problem for folder %s (repo %s) " \
+ "(saved %d; got %d); skipping it. Please see FAQ " \
+ "and manual on how to handle this."% \
+ (folder, folder.getrepository(),
+ folder.get_saveduidvalidity(),
+ folder.get_uidvalidity())
+ self.logger.warning(msg)
def loadmessagelist(self, repos, folder):
- self.logger.debug("Loading message list for %s[%s]"% (
- self.getnicename(repos),
- folder))
+ msg = u"Loading message list for %s[%s]"% (
+ self.getnicename(repos), folder)
+ self.logger.debug(msg)
def messagelistloaded(self, repos, folder, count):
- self.logger.debug("Message list for %s[%s] loaded: %d messages" % (
- self.getnicename(repos), folder, count))
+ self.logger.debug(u"Message list for %s[%s] loaded: %d messages"%
+ (self.getnicename(repos), folder, count))
############################## Message syncing
def syncingmessages(self, sr, srcfolder, dr, dstfolder):
- self.logger.debug("Syncing messages %s[%s] -> %s[%s]" % (
- self.getnicename(sr), srcfolder,
- self.getnicename(dr), dstfolder))
+ msg = u"Syncing messages %s[%s] -> %s[%s]"% (
+ self.getnicename(sr), srcfolder,
+ self.getnicename(dr), dstfolder)
+ self.logger.debug(msg)
def copyingmessage(self, uid, num, num_to_copy, src, destfolder):
- """Output a log line stating which message we copy"""
- self.logger.info("Copy message %s (%d of %d) %s:%s -> %s" % (
+ """Output a log line stating which message we copy. """
+
+ self.logger.info(u"Copy message %s (%d of %d) %s:%s -> %s"% (
uid, num, num_to_copy, src.repository, src,
destfolder.repository))
def deletingmessages(self, uidlist, destlist):
ds = self.folderlist(destlist)
- prefix = "[DRYRUN] " if self.dryrun else ""
- self.info("{0}Deleting {1} messages ({2}) in {3}".format(
+ prefix = u"[DRYRUN] " if self.dryrun else u""
+ self.info(u"{0}Deleting {1} messages ({2}) in {3}".format(
prefix, len(uidlist),
offlineimap.imaputil.uid_sequence(uidlist), ds))
def addingflags(self, uidlist, flags, dest):
- self.logger.info("Adding flag %s to %d messages on %s" % (
- ", ".join(flags), len(uidlist), dest))
+ msg = u"Adding flag %s to %d messages on %s"% (
+ ", ".join(flags), len(uidlist), dest)
+ self.logger.info(msg)
def deletingflags(self, uidlist, flags, dest):
- self.logger.info("Deleting flag %s from %d messages on %s" % (
- ", ".join(flags), len(uidlist), dest))
+ msg = u"Deleting flag %s from %d messages on %s"% (
+ ", ".join(flags), len(uidlist), dest)
+ self.logger.info(msg)
def addinglabels(self, uidlist, label, dest):
- self.logger.info("Adding label %s to %d messages on %s" % (
- label, len(uidlist), dest))
+ msg = u"Adding label %s to %d messages on %s"% (
+ label, len(uidlist), dest)
+ self.logger.info(msg)
def deletinglabels(self, uidlist, label, dest):
- self.logger.info("Deleting label %s from %d messages on %s" % (
- label, len(uidlist), dest))
+ msg = u"Deleting label %s from %d messages on %s"% (
+ label, len(uidlist), dest)
+ self.logger.info(msg)
def settinglabels(self, uid, num, num_to_set, labels, dest):
- self.logger.info("Setting labels to message %d on %s (%d of %d): %s" % (
- uid, dest, num, num_to_set, ", ".join(labels)))
+ msg = u"Setting labels to message %d on %s (%d of %d): %s"% (
+ uid, dest, num, num_to_set, ", ".join(labels))
+ self.logger.info()
def collectingdata(self, uidlist, source):
if uidlist:
- self.logger.info("Collecting data from %d messages on %s"% (
- len(uidlist), source))
+ msg = u"Collecting data from %d messages on %s"% (
+ len(uidlist), source)
else:
- self.logger.info("Collecting data from messages on %s"% source)
+ msg = u"Collecting data from messages on %s"% source
+ self.logger.info(msg)
def serverdiagnostics(self, repository, type):
"""Connect to repository and output useful information for debugging."""
conn = None
- self._msg("%s repository '%s': type '%s'" % (type, repository.name,
- self.getnicename(repository)))
+ self._msg(u"%s repository '%s': type '%s'"% (
+ type, repository.name, self.getnicename(repository)))
try:
if hasattr(repository, 'gethost'): # IMAP
- self._msg("Host: %s Port: %s SSL: %s"% (repository.gethost(),
+ self._msg(u"Host: %s Port: %s SSL: %s"% (repository.gethost(),
repository.getport(), repository.getssl()))
try:
conn = repository.imapserver.acquireconnection()
except OfflineImapError as e:
- self._msg("Failed to connect. Reason %s" % e)
+ self._msg(u"Failed to connect. Reason %s"% e)
else:
if 'ID' in conn.capabilities:
- self._msg("Server supports ID extension.")
+ self._msg(u"Server supports ID extension.")
#TODO: Debug and make below working, it hangs Gmail
#res_type, response = conn.id((
# 'name', offlineimap.__productname__,
# 'version', offlineimap.__bigversion__))
- #self._msg("Server ID: %s %s" % (res_type, response[0]))
- self._msg("Server welcome string: %s" % str(conn.welcome))
- self._msg("Server capabilities: %s\n" % str(conn.capabilities))
+ #self._msg(u"Server ID: %s %s" % (res_type, response[0]))
+ self._msg(u"Server welcome string: %s"% str(conn.welcome))
+ self._msg(u"Server capabilities: %s\n"% str(conn.capabilities))
repository.imapserver.releaseconnection(conn)
if type != 'Status':
folderfilter = repository.getconf('folderfilter', None)
if folderfilter:
- self._msg("folderfilter= %s\n" % folderfilter)
+ self._msg(u"folderfilter= %s\n"% folderfilter)
folderincludes = repository.getconf('folderincludes', None)
if folderincludes:
- self._msg("folderincludes= %s\n" % folderincludes)
+ self._msg(u"folderincludes= %s\n"% folderincludes)
nametrans = repository.getconf('nametrans', None)
if nametrans:
- self._msg("nametrans= %s\n" % nametrans)
+ self._msg(u"nametrans= %s\n"% nametrans)
folders = repository.getfolders()
foldernames = [(f.name, f.getvisiblename(), f.sync_this)
for f in folders]
folders = []
for name, visiblename, sync_this in foldernames:
- syncstr = "" if sync_this else " (disabled)"
- if name == visiblename: folders.append("%s%s" % (name,
- syncstr))
- else: folders.append("%s -> %s%s" % (name,
- visiblename, syncstr))
- self._msg("Folderlist:\n %s\n" % "\n ".join(folders))
+ syncstr = u"" if sync_this else u" (disabled)"
+ if name == visiblename: folders.append(u"%s%s"%
+ (name, syncstr))
+ else: folders.append(u"%s -> %s%s"%
+ (name, visiblename, syncstr))
+ self._msg(u"Folderlist:\n %s\n"% u"\n ".join(folders))
finally:
if conn: #release any existing IMAP connection
repository.imapserver.close()
@@ -467,13 +499,15 @@ class UIBase(object):
################################################## Threads
def getThreadDebugLog(self, thread):
+ thr_name = thread.getName()
+ if globals.options.use_unicode:
+ thr_name = uni.bytes2uni(thr_name)
if thread in self.debugmessages:
- message = "\nLast %d debug messages logged for %s prior to exception:\n"\
- % (len(self.debugmessages[thread]), thread.getName())
- message += "\n".join(self.debugmessages[thread])
+ message = u"\nLast %d debug messages logged for %s prior to" \
+ " exception:\n"% (len(self.debugmessages[thread]), thr_name)
+ message += u"\n".join(self.debugmessages[thread])
else:
- message = "\nNo debug messages were logged for %s."% \
- thread.getName()
+ message = u"\nNo debug messages were logged for %s."% thr_name
return message
def delThreadDebugLog(self, thread):
@@ -481,9 +515,12 @@ class UIBase(object):
del self.debugmessages[thread]
def getThreadExceptionString(self, thread):
- message = "Thread '%s' terminated with exception:\n%s"% \
- (thread.getName(), thread.exit_stacktrace)
- message += "\n" + self.getThreadDebugLog(thread)
+ thr_name = thread.getName()
+ if globals.options.use_unicode:
+ thr_name = uni.bytes2uni(thr_name)
+ message = u"Thread '%s' terminated with exception:\n%s"% (
+ thr_name, thread.exit_stacktrace)
+ message += u"\n" + self.getThreadDebugLog(thread)
return message
def threadException(self, thread):
@@ -508,12 +545,12 @@ class UIBase(object):
self.warn("ERROR: %s"% (exc))
if exc_traceback:
self.warn("\nTraceback:\n%s"% "".join(
- traceback.format_tb(exc_traceback)))
+ traceback.format_tb(exc_traceback)))
if errormsg and errortitle:
self.warn('ERROR: %s\n\n%s\n'% (errortitle, errormsg))
elif errormsg:
- self.warn('%s\n'% errormsg)
+ self.warn('%s\n'% errormsg)
sys.exit(exitstatus)
def threadExited(self, thread):
@@ -528,7 +565,7 @@ class UIBase(object):
def callhook(self, msg):
if self.dryrun:
- self.info("[DRYRUN] {0}".format(msg))
+ self.info(u"[DRYRUN] {0}".format(msg))
else:
self.info(msg)
@@ -566,7 +603,7 @@ class UIBase(object):
if sleepsecs > 0:
if remainingsecs//60 != (remainingsecs-sleepsecs)//60:
- self.logger.debug("Next refresh in %.1f minutes" % (
- remainingsecs/60.0))
+ self.logger.debug(u"Next refresh in %.1f minutes"% (
+ remainingsecs/60.0))
time.sleep(sleepsecs)
return 0
diff --git a/offlineimap/ui/debuglock.py b/offlineimap/ui/debuglock.py
index 673efb0..51a26f9 100644
--- a/offlineimap/ui/debuglock.py
+++ b/offlineimap/ui/debuglock.py
@@ -17,7 +17,15 @@
from threading import Lock, currentThread
import traceback
-logfile = open("/tmp/logfile", "wt")
+import codecs
+
+from offlineimap import globals
+from offlineimap.utils import uni
+
+if globals.options.use_unicode:
+ logfile = codecs.open("/tmp/logfile", "wt", uni.ENCODING)
+else:
+ logfile = open("/tmp/logfile", "wt")
loglock = Lock()
class DebuggingLock:
@@ -26,23 +34,26 @@ class DebuggingLock:
self.name = name
def acquire(self, blocking = 1):
- self.print_tb("Acquire lock")
+ self.print_tb(u"Acquire lock")
self.lock.acquire(blocking)
- self.logmsg("===== %s: Thread %s acquired lock\n"%
+ self.logmsg(u"===== %s: Thread %s acquired lock\n"%
(self.name, currentThread().getName()))
def release(self):
- self.print_tb("Release lock")
+ self.print_tb(u"Release lock")
self.lock.release()
def logmsg(self, msg):
loglock.acquire()
- logfile.write(msg + "\n")
+ msg = msg + "\n"
+ if globals.options.use_unicode:
+ msg = uni.uni2bytes(msg)
+ logfile.write(msg)
logfile.flush()
loglock.release()
def print_tb(self, msg):
- self.logmsg(".... %s: Thread %s attempting to %s\n"% \
+ self.logmsg(u".... %s: Thread %s attempting to %s\n"% \
(self.name, currentThread().getName(), msg) + \
"\n".join(traceback.format_list(traceback.extract_stack())))
diff --git a/offlineimap/utils/const.py b/offlineimap/utils/const.py
index f4584bc..40e4ca9 100644
--- a/offlineimap/utils/const.py
+++ b/offlineimap/utils/const.py
@@ -5,6 +5,7 @@
import copy
+# Assumes variables to be full ASCII. Should be fine while metaprogramming.
class ConstProxy(object):
"""Implements read-only access to a given object
that can be attached to each instance only once."""
@@ -21,12 +22,12 @@ class ConstProxy(object):
def __setattr__(self, name, value):
- raise AttributeError("tried to set '%s' to '%s' for constant object"% \
+ raise AttributeError("tried to set '%s' to '%s' for constant object"%
(name, value))
def __delattr__(self, name):
- raise RuntimeError("tried to delete field '%s' from constant object"% \
+ raise RuntimeError("tried to delete field '%s' from constant object"%
(name))
--
2.2.2
More information about the OfflineIMAP-project
mailing list