Step 1 of rearranging per r129

This commit is contained in:
John Goerzen
2005-04-16 20:05:39 +01:00
parent 3c3b2e1ea7
commit 035fa2a96e
41 changed files with 0 additions and 0 deletions

View File

@@ -1,289 +0,0 @@
# Base folder support
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import __main__
from threading import *
from offlineimap import threadutil
from offlineimap.threadutil import InstanceLimitedThread
class BaseFolder:
def getname(self):
"""Returns name"""
return self.name
def suggeststhreads(self):
"""Returns true if this folder suggests using threads for actions;
false otherwise. Probably only IMAP will return true."""
return 0
def waitforthread(self):
"""For threading folders, waits until there is a resource available
before firing off a thread. For all others, returns immediately."""
pass
def getcopyinstancelimit(self):
"""For threading folders, returns the instancelimitname for
InstanceLimitedThreads."""
raise NotImplementedException
def storesmessages(self):
"""Should be true for any backend that actually saves message bodies.
(Almost all of them). False for the LocalStatus backend. Saves
us from having to slurp up messages just for localstatus purposes."""
return 1
def getvisiblename(self):
return self.name
def getroot(self):
"""Returns the root of the folder, in a folder-specific fashion."""
return self.root
def getsep(self):
"""Returns the separator for this folder type."""
return self.sep
def getfullname(self):
if self.getroot():
return self.getroot() + self.getsep() + self.getname()
else:
return self.getname()
def isuidvalidityok(self, remotefolder):
raise NotImplementedException
def getuidvalidity(self):
raise NotImplementedException
def saveuidvalidity(self, newval):
raise NotImplementedException
def cachemessagelist(self):
"""Reads the message list from disk or network and stores it in
memory for later use. This list will not be re-read from disk or
memory unless this function is called again."""
raise NotImplementedException
def getmessagelist(self):
"""Gets the current message list.
You must call cachemessagelist() before calling this function!"""
raise NotImplementedException
def getmessage(self, uid):
"""Returns the content of the specified message."""
raise NotImplementedException
def savemessage(self, uid, content, flags):
"""Writes a new message, with the specified uid.
If the uid is < 0, the backend should assign a new uid and return it.
If the backend cannot assign a new uid, it returns the uid passed in
WITHOUT saving the message.
IMAP backend should be the only one that can assign a new uid.
If the uid is > 0, the backend should set the uid to this, if it can.
If it cannot set the uid to that, it will save it anyway.
It will return the uid assigned in any case.
"""
raise NotImplementedException
def getmessageflags(self, uid):
"""Returns the flags for the specified message."""
raise NotImplementedException
def savemessageflags(self, uid, flags):
"""Sets the specified message's flags to the given set."""
raise NotImplementedException
def addmessageflags(self, uid, flags):
"""Adds the specified flags to the message's flag set. If a given
flag is already present, it will not be duplicated."""
newflags = self.getmessageflags(uid)
for flag in flags:
if not flag in newflags:
newflags.append(flag)
newflags.sort()
self.savemessageflags(uid, newflags)
def addmessagesflags(self, uidlist, flags):
for uid in uidlist:
self.addmessageflags(uid)
def deletemessageflags(self, uid, flags):
"""Removes each flag given from the message's flag set. If a given
flag is already removed, no action will be taken for that flag."""
newflags = self.getmessageflags(uid)
for flag in flags:
if flag in newflags:
newflags.remove(flag)
newflags.sort()
self.savemessageflags(uid, newflags)
def deletemessage(self, uid):
raise NotImplementedException
def deletemessages(self, uidlist):
for uid in uidlist:
self.deletemessage(uid)
def syncmessagesto_neguid(self, dest, applyto):
"""Pass 1 of folder synchronization.
Look for messages in self with a negative uid. These are messages in
Maildirs that were not added by us. Try to add them to the dests,
and once that succeeds, get the UID, add it to the others for real,
add it to local for real, and delete the fake one."""
for uid in self.getmessagelist().keys():
if uid >= 0:
continue
__main__.ui.copyingmessage(uid, self, applyto)
successobject = None
successuid = None
message = self.getmessage(uid)
flags = self.getmessageflags(uid)
for tryappend in applyto:
successuid = tryappend.savemessage(uid, message, flags)
if successuid > 0:
successobject = tryappend
break
# Did we succeed?
if successobject != None:
# Copy the message to the other remote servers.
for appendserver in [x for x in applyto if x != successobject]:
appendserver.savemessage(successuid, message, flags)
# Copy it to its new name on the local server and delete
# the one without a UID.
self.savemessage(successuid, message, flags)
self.deletemessage(uid)
else:
# Did not find any server to take this message. Ignore.
pass
def copymessageto(self, uid, applyto):
# Sometimes, it could be the case that if a sync takes awhile,
# a message might be deleted from the maildir before it can be
# synced to the status cache. This is only a problem with
# self.getmessage(). So, don't call self.getmessage unless
# really needed.
__main__.ui.copyingmessage(uid, self, applyto)
message = ''
# If any of the destinations actually stores the message body,
# load it up.
for object in applyto:
if object.storesmessages():
message = self.getmessage(uid)
break
flags = self.getmessageflags(uid)
for object in applyto:
newuid = object.savemessage(uid, message, flags)
if newuid > 0 and newuid != uid:
# Change the local uid.
self.savemessage(newuid, message, flags)
self.deletemessage(uid)
uid = newuid
def syncmessagesto_copy(self, dest, applyto):
"""Pass 2 of folder synchronization.
Look for messages present in self but not in dest. If any, add
them to dest."""
threads = []
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages that pass 1 missed.
continue
if not uid in dest.getmessagelist():
if self.suggeststhreads():
self.waitforthread()
thread = InstanceLimitedThread(\
self.getcopyinstancelimit(),
target = self.copymessageto,
name = "Copy message %d from %s" % (uid,
self.getvisiblename()),
args = (uid, applyto))
thread.setDaemon(1)
thread.start()
threads.append(thread)
else:
self.copymessageto(uid, applyto)
for thread in threads:
thread.join()
def syncmessagesto_delete(self, dest, applyto):
"""Pass 3 of folder synchronization.
Look for message present in dest but not in self.
If any, delete them."""
deletelist = []
for uid in dest.getmessagelist().keys():
if uid < 0:
continue
if not uid in self.getmessagelist():
deletelist.append(uid)
if len(deletelist):
__main__.ui.deletingmessages(deletelist, applyto)
for object in applyto:
object.deletemessages(deletelist)
def syncmessagesto_flags(self, dest, applyto):
"""Pass 4 of folder synchronization.
Look for any flag matching issues -- set dest message to have the
same flags that we have."""
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages missed by pass 1
continue
selfflags = self.getmessageflags(uid)
destflags = dest.getmessageflags(uid)
addflags = [x for x in selfflags if x not in destflags]
if len(addflags):
__main__.ui.addingflags(uid, addflags, applyto)
for object in applyto:
object.addmessageflags(uid, addflags)
delflags = [x for x in destflags if x not in selfflags]
if len(delflags):
__main__.ui.deletingflags(uid, delflags, applyto)
for object in applyto:
object.deletemessageflags(uid, delflags)
def syncmessagesto(self, dest, applyto = None):
"""Syncs messages in this folder to the destination.
If applyto is specified, it should be a list of folders (don't forget
to include dest!) to which all write actions should be applied.
It defaults to [dest] if not specified. It is important that
the UID generator be listed first in applyto; that is, the other
applyto ones should be the ones that "copy" the main action."""
if applyto == None:
applyto = [dest]
self.syncmessagesto_neguid(dest, applyto)
self.syncmessagesto_copy(dest, applyto)
self.syncmessagesto_delete(dest, applyto)
# Now, the message lists should be identical wrt the uids present.
# (except for potential negative uids that couldn't be placed
# anywhere)
self.syncmessagesto_flags(dest, applyto)

View File

@@ -1,203 +0,0 @@
# IMAP folder support
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from Base import BaseFolder
from offlineimap import imaputil, imaplib
import rfc822
from StringIO import StringIO
from copy import copy
class IMAPFolder(BaseFolder):
def __init__(self, imapserver, name, visiblename, accountname):
self.name = imaputil.dequote(name)
self.root = imapserver.root
self.sep = imapserver.delim
self.imapserver = imapserver
self.messagelist = None
self.visiblename = visiblename
self.accountname = accountname
def suggeststhreads(self):
return 1
def waitforthread(self):
self.imapserver.connectionwait()
def getcopyinstancelimit(self):
return 'MSGCOPY_' + self.accountname
def getvisiblename(self):
return self.visiblename
def getuidvalidity(self):
imapobj = self.imapserver.acquireconnection()
try:
# Primes untagged_responses
imapobj.select(self.getfullname(), readonly = 1)
return long(imapobj.untagged_responses['UIDVALIDITY'][0])
finally:
self.imapserver.releaseconnection(imapobj)
def cachemessagelist(self):
imapobj = self.imapserver.acquireconnection()
self.messagelist = {}
try:
# Primes untagged_responses
imapobj.select(self.getfullname(), readonly = 1)
maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
if maxmsgid < 1:
# No messages; return
return
# Now, get the flags and UIDs for these.
# We could conceivably get rid of maxmsgid and just say
# '1:*' here.
response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
finally:
self.imapserver.releaseconnection(imapobj)
for messagestr in response:
# Discard the message number.
messagestr = imaputil.imapsplit(messagestr)[1]
options = imaputil.flags2hash(messagestr)
uid = long(options['UID'])
flags = imaputil.flagsimap2maildir(options['FLAGS'])
self.messagelist[uid] = {'uid': uid, 'flags': flags}
def getmessagelist(self):
return self.messagelist
def getmessage(self, uid):
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname(), readonly = 1)
return imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')[1][0][1].replace("\r\n", "\n")
finally:
self.imapserver.releaseconnection(imapobj)
def getmessageflags(self, uid):
return self.messagelist[uid]['flags']
def savemessage(self, uid, content, flags):
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname()) # Needed for search
# This backend always assigns a new uid, so the uid arg is ignored.
# In order to get the new uid, we need to save off the message ID.
message = rfc822.Message(StringIO(content))
mid = imapobj._quote(message.getheader('Message-Id'))
date = imaplib.Time2Internaldate(rfc822.parsedate(message.getheader('Date')))
if content.find("\r\n") == -1: # Convert line endings if not already
content = content.replace("\n", "\r\n")
assert(imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)[0] == 'OK')
# Checkpoint. Let it write out the messages, etc.
assert(imapobj.check()[0] == 'OK')
# Now find the UID it got.
matchinguids = imapobj.uid('search', None,
'(HEADER Message-Id %s)' % mid)[1][0]
matchinguids = matchinguids.split(' ')
matchinguids.sort()
uid = long(matchinguids[-1])
self.messagelist[uid] = {'uid': uid, 'flags': flags}
return uid
finally:
self.imapserver.releaseconnection(imapobj)
def savemessageflags(self, uid, flags):
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname())
result = imapobj.uid('store', '%d' % uid, 'FLAGS',
imaputil.flagsmaildir2imap(flags))
assert result[0] == 'OK', 'Error with store: ' + r[1]
finally:
self.imapserver.releaseconnection(imapobj)
result = result[1][0]
if not result:
self.messagelist[uid]['flags'] = flags
else:
flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def addmessageflags(self, uid, flags):
self.addmessagesflags([uid], flags)
def addmessagesflags(self, uidlist, flags):
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname())
r = imapobj.uid('store',
imaputil.listjoin(uidlist),
'+FLAGS',
imaputil.flagsmaildir2imap(flags))
assert r[0] == 'OK', 'Error with store: ' + r[1]
r = r[1]
finally:
self.imapserver.releaseconnection(imapobj)
# Some IMAP servers do not always return a result. Therefore,
# only update the ones that it talks about, and manually fix
# the others.
needupdate = copy(uidlist)
for result in r:
if result == None:
# Compensate for servers that don't return anything from
# STORE.
continue
attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
if not ('UID' in attributehash and 'FLAGS' in attributehash):
# Compensate for servers that don't return a UID attribute.
continue
flags = attributehash['FLAGS']
uid = long(attributehash['UID'])
self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
try:
needupdate.remove(uid)
except ValueError: # Let it slide if it's not in the list
pass
for uid in needupdate:
for flag in flags:
if not flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].append(flag)
self.messagelist[uid]['flags'].sort()
def deletemessage(self, uid):
self.deletemessages([uid])
def deletemessages(self, uidlist):
# Weed out ones not in self.messagelist
uidlist = [uid for uid in uidlist if uid in self.messagelist]
if not len(uidlist):
return
self.addmessagesflags(uidlist, ['T'])
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname())
assert(imapobj.expunge()[0] == 'OK')
finally:
self.imapserver.releaseconnection(imapobj)
for uid in uidlist:
del self.messagelist[uid]

View File

@@ -1,102 +0,0 @@
# Local status cache virtual folder
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from Base import BaseFolder
import os
magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1"
class LocalStatusFolder(BaseFolder):
def __init__(self, root, name):
self.name = name
self.root = root
self.sep = '.'
self.filename = os.path.join(root, name)
self.messagelist = None
def storesmessages(self):
return 0
def isnewfolder(self):
return not os.path.exists(self.filename)
def getname(self):
return self.name
def getroot(self):
return self.root
def getsep(self):
return self.sep
def getfullname(self):
return self.filename
def cachemessagelist(self):
if self.isnewfolder():
self.messagelist = {}
return
file = open(self.filename, "rt")
self.messagelist = {}
line = file.readline().strip()
assert(line == magicline)
for line in file.xreadlines():
line = line.strip()
uid, flags = line.split(':')
uid = long(uid)
flags = [x for x in flags]
self.messagelist[uid] = {'uid': uid, 'flags': flags}
file.close()
def save(self):
file = open(self.filename + ".tmp", "wt")
file.write(magicline + "\n")
for msg in self.messagelist.values():
flags = msg['flags']
flags.sort()
flags = ''.join(flags)
file.write("%s:%s\n" % (msg['uid'], flags))
file.close()
os.rename(self.filename + ".tmp", self.filename)
def getmessagelist(self):
return self.messagelist
def savemessage(self, uid, content, flags):
if uid < 0:
# We cannot assign a uid.
return uid
if uid in self.messagelist: # already have it
self.savemessageflags(uid, flags)
return uid
self.messagelist[uid] = {'uid': uid, 'flags': flags}
return uid
def getmessageflags(self, uid):
return self.messagelist[uid]['flags']
def savemessageflags(self, uid, flags):
self.messagelist[uid]['flags'] = flags
def deletemessage(self, uid):
if not uid in self.messagelist:
return
del(self.messagelist[uid])

View File

@@ -1,211 +0,0 @@
# Maildir folder support
# Copyright (C) 2002 John Goerzen
# <jgoerzen@complete.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from Base import BaseFolder
from offlineimap import imaputil
import os.path, os, re, time, socket, md5
timeseq = 0
lasttime = long(0)
def gettimeseq():
global lasttime, timeseq
thistime = long(time.time())
if thistime == lasttime:
timeseq += 1
return timeseq
else:
lasttime = long(time.time())
timeseq = 0
return timeseq
class MaildirFolder(BaseFolder):
def __init__(self, root, name):
self.name = name
self.root = root
self.sep = '.'
self.uidfilename = os.path.join(self.getfullname(), "offlineimap.uidvalidity")
self.messagelist = None
def getfullname(self):
return os.path.join(self.getroot(), self.getname())
def getuidvalidity(self):
if not os.path.exists(self.uidfilename):
return None
file = open(self.uidfilename, "rt")
retval = long(file.readline().strip())
file.close()
return retval
def saveuidvalidity(self, newval):
file = open(self.uidfilename, "wt")
file.write("%d\n" % newval)
file.close()
def isuidvalidityok(self, remotefolder):
myval = self.getuidvalidity()
if myval != None:
return myval == remotefolder.getuidvalidity()
else:
self.saveuidvalidity(remotefolder.getuidvalidity())
return 1
def _scanfolder(self):
"""Cache the message list. Maildir flags are:
R (replied)
S (seen)
T (trashed)
D (draft)
F (flagged)
and must occur in ASCII order."""
retval = {}
files = []
nouidcounter = -1 # Messages without UIDs get
# negative UID numbers.
for dirannex in ['new', 'cur']:
fulldirname = os.path.join(self.getfullname(), dirannex)
files.extend([os.path.join(fulldirname, filename) for
filename in os.listdir(fulldirname)])
for file in files:
messagename = os.path.basename(file)
foldermatch = re.search(',FMD5=([0-9a-f]{32})', messagename)
if (not foldermatch) or \
md5.new(self.getvisiblename()).hexdigest() \
!= foldermatch.group(1):
# If there is no folder MD5 specified, or if it mismatches,
# assume it is a foreign (new) message and generate a
# negative uid for it
uid = nouidcounter
nouidcounter -= 1
else: # It comes from our folder.
uidmatch = re.search(',U=(\d+)', messagename)
uid = None
if not uidmatch:
uid = nouidcounter
nouidcounter -= 1
else:
uid = long(uidmatch.group(1))
flagmatch = re.search(':.*2,([A-Z]+)', messagename)
flags = []
if flagmatch:
flags = [x for x in flagmatch.group(1)]
flags.sort()
if 'T' in flags:
# Message is marked for deletion; just delete it now.
# Otherwise, the T flag will be propogated to the IMAP
# server, and then expunged there, and then deleted here.
# Might as well just delete it now, to help make things
# more robust.
os.unlink(file)
else:
retval[uid] = {'uid': uid,
'flags': flags,
'filename': file}
return retval
def cachemessagelist(self):
self.messagelist = self._scanfolder()
def getmessagelist(self):
return self.messagelist
def getmessage(self, uid):
filename = self.getmessagelist()[uid]['filename']
file = open(filename, 'rt')
retval = file.read()
file.close()
return retval
def savemessage(self, uid, content, flags):
if uid < 0:
# We cannot assign a new uid.
return uid
if uid in self.getmessagelist():
# We already have it.
self.savemessageflags(uid, flags)
return uid
newdir = os.path.join(self.getfullname(), 'new')
tmpdir = os.path.join(self.getfullname(), 'tmp')
messagename = None
attempts = 0
while 1:
if attempts > 15:
raise IOError, "Couldn't write to file %s" % messagename
messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \
(long(time.time()),
gettimeseq(),
os.getpid(),
socket.gethostname(),
uid,
md5.new(self.getvisiblename()).hexdigest())
if os.path.exists(os.path.join(tmpdir, messagename)):
time.sleep(2)
attempts += 1
else:
break
file = open(os.path.join(tmpdir, messagename), "wt")
file.write(content)
file.close()
os.link(os.path.join(tmpdir, messagename),
os.path.join(newdir, messagename))
os.unlink(os.path.join(tmpdir, messagename))
self.messagelist[uid] = {'uid': uid, 'flags': [],
'filename': os.path.join(newdir, messagename)}
self.savemessageflags(uid, flags)
return uid
def getmessageflags(self, uid):
return self.getmessagelist()[uid]['flags']
def savemessageflags(self, uid, flags):
oldfilename = self.getmessagelist()[uid]['filename']
newpath, newname = os.path.split(oldfilename)
infostr = ':'
infomatch = re.search('(:.*)$', newname)
if infomatch: # If the info string is present..
infostr = infomatch.group(1)
newname = newname.split(':')[0] # Strip off the info string.
infostr = re.sub('2,[A-Z]*', '', infostr)
flags.sort()
infostr += '2,' + ''.join(flags)
newname += infostr
newfilename = os.path.join(newpath, newname)
if (newfilename != oldfilename):
os.rename(oldfilename, newfilename)
self.getmessagelist()[uid]['flags'] = flags
self.getmessagelist()[uid]['filename'] = newfilename
def getmessageflags(self, uid):
return self.getmessagelist()[uid]['flags']
def deletemessage(self, uid):
if not uid in self.messagelist:
return
filename = self.getmessagelist()[uid]['filename']
try:
os.unlink(filename)
except IOError:
# Can't find the file -- maybe already deleted?
newmsglist = self._scanfolder()
if uid in newmsglist: # Nope, try new filename.
os.unlink(newmsglist[uid]['filename'])
# Yep -- return.
del(self.messagelist[uid])

View File

@@ -1,2 +0,0 @@
import Base, IMAP, Maildir, LocalStatus