[PATCH/RFC 6/7] Main update method. Loop through each account, and prepare for update.

Hubert Pineault hpineault at riseup.net
Sun Mar 3 08:57:02 GMT 2019


  For the update process, we use two SyncableAccount instance, one for
  original config file (oldaccount) and one for the new config file
  (newaccount). Before initializing newaccount, we need to move
  oldaccount maildir root and metadata folder to a backup location. So
  we first initialize old account to get folder names and we move
  them. We then initialize newaccount, maildir and metadata folders
  will be created automatically. Once both accounts are initialized,
  we sync folder structure for newaccount and then call
  newaccount.get_content_from_account() to procede to copying or
  moving content from old folder structure to new folder structure.

  The whole update process should be rewritten in a new class
  UpdatableAccount inheriting from Account. The UpdatableAccount class
  should have these repos defined: statusrepos, oldlocalrepos,
  newlocalrepos, remoterepos. To avoid having to pass two imap list
  command and two independent remoterepos, we will likely have to write
  a new method to replace SyncableAccount.get_local_folder(remotefolder)
  which seems incompatible with two sets of nametrans.

Signed-off-by: Hubert Pineault <hpineault at riseup.net>
---
 offlineimap/folder/Maildir.py |  67 +++++++++++++++
 offlineimap/init.py           | 188 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 255 insertions(+)

diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index 795f4d3..bab8284 100644
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -551,3 +551,70 @@ class MaildirFolder(BaseFolder):
                 self.ui.warn(("Inconsistent FMD5 for file `%s':"
                               " Neither `%s' nor `%s' found")
                              % (filename, oldfmd5, self._foldermd5))
+
+    def sendcontentto(self, dstfolder, movecontent=False, dryrun=False):
+        from shutil import move, copy2
+
+        srcfolder = self
+        folderpath = os.path.join(srcfolder.root, srcfolder.name)
+
+        for d in os.listdir(folderpath):
+            if os.path.isdir(os.path.join(folderpath, d)):
+                messages = os.listdir(os.path.join(folderpath, d))
+                num = 0
+                totalnum = len(messages)
+                self.ui.nbmessagestosend(os.path.join(dstfolder.name, d),
+                                         totalnum, movecontent)
+
+                for f in messages:
+                    #TODO: Show progression
+
+                    if os.path.isdir(os.path.join(folderpath, d, f)):
+                        '''There should not be any folder in here. If there
+                        is, should we also deal with it? In a sane folder
+                        structure, this should not happen. Or am I wrong?'''
+
+                        self.ui.info("A folder was found in source folder '{0}'"
+                                     "while trying to move its content. Ignoring"
+                                     "'{1}'. This indicate that the original"
+                                     "folder structure is probably corrupt in"
+                                     "some way. Please investigate.",
+                                     format(os.path.join(srcfolder.name, d), f))
+                        continue
+
+                    dstfile = os.path.join(dstfolder.root, dstfolder.name, d, f)
+                    if movecontent:
+                        #TODO: Prevent or ask file overwrite
+                        move(os.path.join(folderpath, d, f),
+                             dstfile)
+                    else:
+                        num += 1
+                        if os.path.exists(dstfile):
+                            self.ui.ignorecopyingmessage(f, srcfolder, dstfolder)
+                            continue
+                        if not dryrun:
+                            copy2(os.path.join(folderpath, d, f),
+                                  dstfile)
+
+                if (movecontent
+                    and len(os.listdir(os.path.join(folderpath, d))) == 0
+                        and not dryrun):
+                    os.rmdir(os.path.join(folderpath, d))
+                elif movecontent and not dryrun:
+                    self.ui.info("Folder {0} is not empty. Some files were not moved".
+                                 format(os.path.join(folderpath, d)))
+
+            else:
+                '''If there is a file in srcfolder, should we also deal with it? In a 
+                sane folder structure, this should not happen. Or am I wrong?
+                move(os.path.join(folderpath, d),
+                     os.join(dstfolder.root, dstfolder.name, f))'''
+                self.ui.ignorecopyingmessage(self._parse_filename(d)['UID'],
+                                             srcfolder, dstfolder)
+
+        if movecontent and len(os.listdir(folderpath)) == 0 \
+           and not dryrun:
+            os.rmdir(folderpath)
+        elif movecontent and not dryrun:
+            self.ui.info("Folder {0} is not empty. Some files were not moved.".
+                         format(folderpath))
diff --git a/offlineimap/init.py b/offlineimap/init.py
index e82d808..b31f7d1 100644
--- a/offlineimap/init.py
+++ b/offlineimap/init.py
@@ -628,6 +628,194 @@ class OfflineImap(object):
                 prof.dump_stats(os.path.join(
                     profiledir, "%s_%s.prof"% (dt, account.getname())))
 
+    def __updateconf(self, list_oldaccounts, list_newaccounts, profiledir):
+        """Executed only in singlethreaded mode.
+
+        For each account found in new config source file,
+        get folder structure from both files and change
+        local repository accordingly.
+
+        Assumes that list_newaccounts are only existing account
+        in old config file, and that config_filename and newconfig_filename
+        are defined.
+
+        :param accs: A list of accounts that should be synced
+        """
+        if profiledir:
+            self.ui.error("Profile mode in config update is not implemented yet!")
+            # Profile mode.
+            raise NotImplementedError
+            
+        self.ui.updateconf(self.config_filename, self.newconfig_filename)
+        
+        # For each account in new config file, initiate both
+        # account (old and new). Then set a temporary dir for
+        # the new folder structure
+        for accountname in list_newaccounts:
+            updatedone = False
+            self.ui.updateconfacct(accountname)
+
+            try:
+                # Disable remote folder creation
+                conf_account_remoterepos = 'Repository ' + \
+                                           self.newconfig.get("Account " + accountname,
+                                                              'remoterepository')
+                self.config.set(conf_account_remoterepos,
+                                'createfolders',
+                                'False')
+                self.newconfig.set(conf_account_remoterepos,
+                                'createfolders',
+                                'False')
+                localrepos = 'Repository ' + \
+                             self.config.get("Account " + accountname,
+                                             'localrepository')
+                newconf_localfolders = os.path.expanduser(
+                    self.newconfig.get(localrepos, 'localfolders'))
+                failedupdaterestore = os.path.expanduser(
+                    self.newconfig.get('general', 'failedupdaterestore'))
+
+                # Set temporary dir for dryrun. Otherwise, old and new accounts
+                # will point to the same dir.
+                if self.newconfig.getboolean('general', 'dry-run'):
+                    if failedupdaterestore:
+                        newtmpmetadatadir = failedupdaterestore                            
+                    else:
+                        tmpfolder_name = '.tmp-update-dryrun'
+                        metadatadir = os.path.expanduser(self.newconfig.
+                                                            getdefault("general",
+                                                                       "metadata",
+                                                                       "~/.offlineimap"))
+                        tmpfolder = os.path.join(metadatadir,
+                                                 tmpfolder_name)
+                        if not os.path.exists(tmpfolder):
+                            os.makedirs(tmpfolder, 0o700)
+                        newtmplocalfolders = os.path.join(metadatadir,
+                                                          tmpfolder,
+                                                          os.path.basename(newconf_localfolders))
+
+                        newconf_localrepos = 'Repository ' + \
+                             self.config.get("Account " + accountname,
+                                             'localrepository')
+                        self.newconfig.set(newconf_localrepos,
+                                           'localfolders',
+                                           newtmplocalfolders)
+                        newtmpmetadatadir = os.path.join(metadatadir,
+                                                         tmpfolder,
+                                                         'metadata')
+                    self.newconfig.set('general', 'metadata', newtmpmetadatadir)
+
+                threading.currentThread().name = \
+                        "Account getfolder %s"% accountname
+
+                ### Old account
+                oldaccount = accounts.SyncableAccount(self.config,
+                                                      accountname)
+                oldaccount.syncrunner()
+
+                ### Proceed to update
+                # Check for CTRL-C or SIGTERM (not sure if it's ok).
+                if oldaccount.abort_NOW_signal.is_set():
+                    break
+
+                # Backup metadata and point old account to it
+                metadatadir = os.path.expanduser(oldaccount.metadatadir)
+                metadatabak = os.path.join(metadatadir,
+                                           "UpdateBackup_" + accountname)
+                oldaccount.movemetadatadir(metadatabak)
+
+                # Move oldlocalrepo to a backup folder
+                from datetime import datetime
+                updatetime = datetime.today().strftime('%y%m%d.%H%M')
+                maildir = oldaccount.localrepos.root
+                mailbak = self.getbackupname(
+                    os.path.join(metadatabak, '{0}.{1}'.format(
+                        os.path.basename(maildir), updatetime)))
+                oldaccount.localrepos.moveroot(mailbak)
+                oldaccount.localrepos.forgetfolders()
+                oldaccount.localrepos.getfolders()
+
+                ### New account
+                if failedupdaterestore \
+                   and not self.newconfig.getboolean('general', 'dry-run'):
+                    from shutil import move
+                    move(os.path.join(failedupdaterestore,
+                                      os.path.basename(newconf_localfolders)),
+                         newconf_localfolders)
+                    for item in os.listdir(failedupdaterestore):
+                        s = os.path.join(failedupdaterestore, item)
+                        d = os.path.join(metadatadir, item)
+                        move(s, d)
+                    os.rmdir(failedupdaterestore)
+
+                newaccount = accounts.SyncableAccount(self.newconfig,
+                                                      accountname)
+                newaccount.syncrunner()
+
+                # Check for CTRL-C or SIGTERM (not sure if it's ok).
+                if newaccount.abort_NOW_signal.is_set():
+                    break
+
+                # Create the new folder structure
+                newaccount.remoterepos.sync_folder_structure(newaccount.localrepos,
+                                                             newaccount.statusrepos)
+                # Moving old content into new structure
+                newaccount.get_content_from_account(oldaccount)
+
+            except:
+                try:
+                    newaccount
+                except:
+                    pass
+                else:
+                    # Backup failed update metadata
+                    failedmetadatabak = self.getbackupname(
+                        os.path.join(os.path.expanduser(newaccount.metadatadir),
+                                     "FailedUpdate_{0}.{1}".
+                                     format(accountname, updatetime)))
+                    newaccount.movemetadatadir(failedmetadatabak)
+                    # Backup failed update maildir
+                    mailbasename = os.path.basename(newaccount.localrepos.root)
+                    failedmaildirbak = os.path.join(failedmetadatabak, mailbasename)
+                    newaccount.localrepos.moveroot(failedmaildirbak)
+                try:
+                    oldaccount
+                except:
+                    pass
+                else:
+                    # Move back old account maildir and metadata to their original dir
+                    if oldaccount.metadatadir != metadatadir:
+                        oldaccount.movemetadatadir(metadatadir)
+                    try:
+                        oldaccount.localrepos
+                    except:
+                        pass
+                    else:
+                        if oldaccount.localrepos.root != maildir:
+                            oldaccount.localrepos.moveroot(maildir)
+                raise
+
+            finally:
+                oldaccount._unlock()
+                newaccount._unlock()
+                if self.newconfig.getboolean('general', 'dry-run') \
+                   and os.path.exists(tmpfolder):
+                    from shutil import rmtree
+                    try:
+                        rmtree(tmpfolder)
+                    except IOError:
+                        raise #TODO Message error
+                    self.ui.updateconfacctdone(accountname)
+
+        self.ui.updateconfdone()
+
+    def getbackupname(self, folder):
+        i = 0
+        while os.path.exists(folder):
+            i += 1
+            folder = '{0}.{1}'.format(folder, i)
+        return folder
+
+
     def __serverdiagnostics(self, options):
         self.ui.info("  imaplib2: %s (%s)"% (imaplib.__version__, imaplib.DESC))
         for accountname in self._get_activeaccounts(options):
-- 
2.11.0




More information about the OfflineIMAP-project mailing list