From e023f190b0eed84fefc10e28bfe5e4e0467ff257 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 10 Jun 2011 17:32:40 +0200 Subject: [PATCH] folder/Maildir: Store only relative filename components MaildirFolder.messagelist[*]['filename'] was storing the absolute file paths for all stored emails. While this is convenient, it wastes much space, as the folder prefix is always the same and it is known to the MaildirFolder. Just 40 chars in a folder with 100k mails waste >4MB of space. Adapt the few locations where we need the full path to construct it dynamically. Signed-off-by: Sebastian Spaeth Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/Maildir.py | 47 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index b9f9d05..a798b36 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -185,27 +185,32 @@ class MaildirFolder(BaseFolder): return self.messagelist def getmessage(self, uid): + """Return the content of the message""" filename = self.messagelist[uid]['filename'] - file = open(filename, 'rt') + filepath = os.path.join(self.getfullname(), filename) + file = open(filepath, 'rt') retval = file.read() file.close() + #TODO: WHY are we replacing \r\n with \n here? And why do we + # read it as text? return retval.replace("\r\n", "\n") def getmessagetime( self, uid ): filename = self.messagelist[uid]['filename'] - st = os.stat(filename) + filepath = os.path.join(self.getfullname(), filename) + st = os.stat(filepath) return st.st_mtime def savemessage(self, uid, content, flags, rtime): # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. - self.ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \ - (repr(flags), repr(content))) + self.ui.debug('maildir', 'savemessage: called to write with flags %s ' + 'and content %s' % (repr(flags), repr(content))) if uid < 0: # We cannot assign a new uid. return uid if uid in self.messagelist: - # We already have it. + # We already have it, just update flags. self.savemessageflags(uid, flags) return uid @@ -231,7 +236,8 @@ class MaildirFolder(BaseFolder): else: break tmpmessagename = messagename.split(',')[0] - self.ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename) + self.ui.debug('maildir', 'savemessage: using temporary name %s' %\ + tmpmessagename) file = open(os.path.join(tmpdir, tmpmessagename), "wt") file.write(content) @@ -239,10 +245,10 @@ class MaildirFolder(BaseFolder): file.flush() if self.dofsync: os.fsync(file.fileno()) - file.close() + if rtime != None: - os.utime(os.path.join(tmpdir,tmpmessagename), (rtime,rtime)) + os.utime(os.path.join(tmpdir, tmpmessagename), (rtime, rtime)) self.ui.debug('maildir', 'savemessage: moving from %s to %s' % \ (tmpmessagename, messagename)) if tmpmessagename != messagename: # then rename it @@ -259,7 +265,8 @@ class MaildirFolder(BaseFolder): pass self.messagelist[uid] = {'uid': uid, 'flags': [], - 'filename': os.path.join(tmpdir, messagename)} + 'filename': os.path.join('tmp', messagename)} + # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) self.ui.debug('maildir', 'savemessage: returning uid %d' % uid) return uid @@ -269,14 +276,14 @@ class MaildirFolder(BaseFolder): def savemessageflags(self, uid, flags): oldfilename = self.messagelist[uid]['filename'] - newpath, newname = os.path.split(oldfilename) + dir_prefix, newname = os.path.split(oldfilename) tmpdir = os.path.join(self.getfullname(), 'tmp') if 'S' in flags: # If a message has been seen, it goes into the cur - # directory. CR debian#152482, [complete.org #4] - newpath = os.path.join(self.getfullname(), 'cur') + # directory. CR debian#152482 + dir_prefix = 'cur' else: - newpath = os.path.join(self.getfullname(), 'new') + dir_prefix = 'new' infostr = ':' infomatch = re.search('(:.*)$', newname) if infomatch: # If the info string is present.. @@ -287,15 +294,16 @@ class MaildirFolder(BaseFolder): infostr += '2,' + ''.join(flags) newname += infostr - newfilename = os.path.join(newpath, newname) + newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): - os.rename(oldfilename, newfilename) + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), newfilename)) self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename # By now, the message had better not be in tmp/ land! final_dir, final_name = os.path.split(self.messagelist[uid]['filename']) - assert final_dir != tmpdir + assert final_dir != 'tmp' def deletemessage(self, uid): """Unlinks a message file from the Maildir. @@ -309,13 +317,16 @@ class MaildirFolder(BaseFolder): return filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) try: - os.unlink(filename) + os.unlink(filepath) except OSError: # Can't find the file -- maybe already deleted? newmsglist = self._scanfolder() if uid in newmsglist: # Nope, try new filename. - os.unlink(newmsglist[uid]['filename']) + filename = newmsglist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + os.unlink(filepath) # Yep -- return. del(self.messagelist[uid])