Properly manipulate contents of messagelist for folder

Create initializer function that puts default values to all fields
of message list item.  Fix all code that directly assigns some hash
to the elements of messagelist: for direct assignments only initializer
is now permitted, all other modification are done in-place.

Signed-off-by: Eygene Ryabinkin <rea@codelabs.ru>

offlineimap/folder/Base.py

@@ -236,6 +236,17 @@ class BaseFolder(object):
         You must call cachemessagelist() before calling this function!"""
         raise NotImplementedError
 
+    def msglist_item_initializer(self, uid):
+        """
+        Returns value for empty messagelist element with given UID.
+
+        This function must initialize all fields of messagelist item
+        and must be called every time when one creates new messagelist
+        entry to ensure that all fields that must be present are present.
+
+        """
+        raise NotImplementedError
+
     def uidexists(self, uid):
         """Returns True if uid exists"""
         return uid in self.getmessagelist()

offlineimap/folder/Gmail.py

@@ -110,6 +110,11 @@ class GmailFolder(IMAPFolder):
         else:
             return set()
 
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0}
+
+
     # TODO: merge this code with the parent's cachemessagelist:
     # TODO: they have too much common logics.
     def cachemessagelist(self):
@@ -152,6 +157,7 @@ class GmailFolder(IMAPFolder):
                                           minor = 1)
             else:
                 uid = long(options['UID'])
+                self.messagelist[uid] = self.msglist_item_initializer(uid)
                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
                 m = re.search('\(([^\)]*)\)', options['X-GM-LABELS'])
                 if m:

offlineimap/folder/GmailMaildir.py

@@ -56,6 +56,12 @@ class GmailMaildirFolder(MaildirFolder):
                 return True
         return False  #Nope, nothing changed
 
+
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'flags': set(), 'labels': set(), 'filename': '/no-dir/no-such-file/', 'mtime': 0}
+
+
     def cachemessagelist(self):
         if self.messagelist is None:
             self.messagelist = self._scanfolder()

offlineimap/folder/IMAP.py

@@ -202,6 +202,10 @@ class IMAPFolder(BaseFolder):
 
         return msgsToFetch
 
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'uid': uid, 'flags': set(), 'time': 0}
+
 
     # Interface from BaseFolder
     def cachemessagelist(self):
@@ -239,6 +243,7 @@ class IMAPFolder(BaseFolder):
                                           minor = 1)
             else:
                 uid = long(options['UID'])
+                self.messagelist[uid] = self.msglist_item_initializer(uid)
                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
                 rtime = imaplibutil.Internaldate2epoch(messagestr)
                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
@@ -641,7 +646,8 @@ class IMAPFolder(BaseFolder):
             if imapobj: self.imapserver.releaseconnection(imapobj)
 
         if uid: # avoid UID FETCH 0 crash happening later on
-            self.messagelist[uid] = {'uid': uid, 'flags': flags}
+            self.messagelist[uid] = self.msglist_item_initializer(uid)
+            self.messagelist[uid]['flags'] = flags
 
         self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
         return uid

offlineimap/folder/LocalStatus.py

@@ -57,6 +57,11 @@ class LocalStatusFolder(BaseFolder):
             os.unlink(self.filename)
 
 
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0}
+
+
     def readstatus_v1(self, fp):
         """
         Read status folder in format version 1.
@@ -76,7 +81,8 @@ class LocalStatusFolder(BaseFolder):
                     (line, self.filename)
                 self.ui.warn(errstr)
                 raise ValueError(errstr)
-            self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': 0, 'labels': set()}
+            self.messagelist[uid] = self.msglist_item_initializer(uid)
+            self.messagelist[uid]['flags'] = flags
 
 
     def readstatus(self, fp):
@@ -100,7 +106,10 @@ class LocalStatusFolder(BaseFolder):
                     (line, self.filename)
                 self.ui.warn(errstr)
                 raise ValueError(errstr)
-            self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': mtime, 'labels': labels}
+            self.messagelist[uid] = self.msglist_item_initializer(uid)
+            self.messagelist[uid]['flags'] = flags
+            self.messagelist[uid]['mtime'] = mtime
+            self.messagelist[uid]['labels'] = labels
 
 
     # Interface from BaseFolder
@@ -197,7 +206,11 @@ class LocalStatusFolder(BaseFolder):
             self.savemessageflags(uid, flags)
             return uid
 
-        self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels}
+        self.messagelist[uid] = self.msglist_item_initializer(uid)
+        self.messagelist[uid]['flags'] = flags
+        self.messagelist[uid]['time'] = rtime
+        self.messagelist[uid]['mtime'] = mtime
+        self.messagelist[uid]['labels'] = labels
         self.save()
         return uid
 

offlineimap/folder/LocalStatusSQLite.py

@@ -184,14 +184,23 @@ class LocalStatusSQLiteFolder(BaseFolder):
         self._newfolder = True
 
 
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'uid': uid, 'flags': set(), 'labels': set(), 'time': 0, 'mtime': 0}
+
+
     # Interface from BaseFolder
     def cachemessagelist(self):
         self.messagelist = {}
         cursor = self.connection.execute('SELECT id,flags,mtime,labels from status')
         for row in cursor:
+            uid = row[0]
+            self.messagelist[uid] = self.msglist_item_initializer(uid)
             flags = set(row[1])
             labels = set([lb.strip() for lb in row[3].split(',') if len(lb.strip()) > 0])
-            self.messagelist[row[0]] = {'uid': row[0], 'flags': flags, 'mtime': row[2], 'labels': labels}
+            self.messagelist[uid]['flags'] = flags
+            self.messagelist[uid]['labels'] = labels
+            self.messagelist[uid]['mtime'] = row[2]
 
     # Interface from LocalStatusFolder
     def save(self):
@@ -271,6 +280,7 @@ class LocalStatusSQLiteFolder(BaseFolder):
             self.savemessageflags(uid, flags)
             return uid
 
+        self.messagelist[uid] = self.msglist_item_initializer(uid)
         self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels}
         flags = ''.join(sorted(flags))
         labels = ', '.join(sorted(labels))
@@ -278,9 +288,11 @@ class LocalStatusSQLiteFolder(BaseFolder):
                          (uid,flags,mtime,labels))
         return uid
 
+
     # Interface from BaseFolder
     def savemessageflags(self, uid, flags):
-        self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        assert self.uidexists(uid)
+        self.messagelist[uid]['flags'] = flags
         flags = ''.join(sorted(flags))
         self.__sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid))
 

offlineimap/folder/Maildir.py

@@ -191,7 +191,9 @@ class MaildirFolder(BaseFolder):
                 else:
                     uid = long(uidmatch.group(1))
             # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S
-            retval[uid] = {'flags': flags, 'filename': filepath}
+            retval[uid] = self.msglist_item_initializer(uid)
+            retval[uid]['flags'] = flags
+            retval[uid]['filename'] = filepath
         return retval
 
     # Interface from BaseFolder
@@ -208,6 +210,12 @@ class MaildirFolder(BaseFolder):
                 return True
         return False  #Nope, nothing changed
 
+
+    # Interface from BaseFolder
+    def msglist_item_initializer(self, uid):
+        return {'flags': set(), 'filename': '/no-dir/no-such-file/'}
+
+
     # Interface from BaseFolder
     def cachemessagelist(self):
         if self.messagelist is None:
@@ -319,7 +327,9 @@ class MaildirFolder(BaseFolder):
         if rtime != None:
             os.utime(os.path.join(self.getfullname(), tmpname), (rtime, rtime))
 
-        self.messagelist[uid] = {'flags': flags, 'filename': tmpname}
+        self.messagelist[uid] = self.msglist_item_initializer(uid)
+        self.messagelist[uid]['flags'] = flags
+        self.messagelist[uid]['filename'] = tmpname
         # savemessageflags moves msg to 'cur' or 'new' as appropriate
         self.savemessageflags(uid, flags)
         self.ui.debug('maildir', 'savemessage: returning uid %d' % uid)
@@ -339,6 +349,9 @@ class MaildirFolder(BaseFolder):
         Note that this function does not check against dryrun settings,
         so you need to ensure that it is never called in a
         dryrun mode."""
+
+        assert uid in self.messagelist
+
         oldfilename = self.messagelist[uid]['filename']
         dir_prefix, filename = os.path.split(oldfilename)
         # If a message has been seen, it goes into 'cur'