diff --git a/debian/changelog b/debian/changelog index db3f174..364afdb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ offlineimap (6.2.0) unstable; urgency=low * FIX: Removed imaplib2 entirely. It was the cause of numerous - crash/hang bugs. + crash/hang bugs. Closes: #536156, #536329. * REMOVAL: IDLE support had to be removed because it depended on imaplib2. * Do not request INTERNALDATE when it is not needed. diff --git a/offlineimap.conf b/offlineimap.conf index 1a32da6..7a1667e 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -195,6 +195,30 @@ remoterepository = RemoteExample # You can also specify parameters to the commands # presynchook = imapfilter -c someotherconfig.lua +# If you have a limited amount of bandwidth available you can exclude larger +# messages (e.g. those with large attachments etc). If you do this it +# will appear to offlineimap that these messages do not exist at all. They +# will not be copied, have flags changed etc. For this to work on an IMAP +# server the server must have server side search enabled. This works with gmail +# and most imap servers (e.g. cyrus etc) +# The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB + +# maxsize = 2000000 + + +# When you are starting to sync an already existing account yuo can tell offlineimap +# to sync messages from only the last x days. When you do this messages older than x +# days will be completely ignored. This can be useful for importing existing accounts +# when you do not want to download large amounts of archive email. + +# Messages older than maxage days will not be synced, their flags will +# not be changed, they will not be deleted etc. For offlineimap it will be like these +# messages do not exist. This will perform an IMAP search in the case of IMAP or Gmail +# and therefor requires that the server support server side searching. This will +# calculate the earliest day that would be included in the search and include all +# messages from that day until today. e.g. maxage = 3 to sync only the last 3 days mail + +# maxage = 3 [Repository LocalExample] diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 55fba9b..e7f1324 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -178,16 +178,22 @@ class AccountSynchronizationMixin: # Connect to the local cache. self.statusrepos = offlineimap.repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self) - + + #might need changes here to ensure that one account sync does not crash others... if not self.refreshperiod: + self.sync(siglistener) self.ui.acctdone(self.name) + return + + looping = 1 while looping: self.sync(siglistener) looping = self.sleeper(siglistener) != 2 - self.ui.acctdone(self.name) + self.ui.acctdone(self.name) + def getaccountmeta(self): return os.path.join(self.metadatadir, 'Account-' + self.name) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index bb25ed9..c90d0e5 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -24,6 +24,7 @@ from offlineimap.version import versionstr import rfc822, time, string, random, binascii, re from StringIO import StringIO from copy import copy +import time class IMAPFolder(BaseFolder): @@ -116,6 +117,7 @@ class IMAPFolder(BaseFolder): return False + # TODO: Make this so that it can define a date that would be the oldest messages etc. def cachemessagelist(self): imapobj = self.imapserver.acquireconnection() self.messagelist = {} @@ -123,20 +125,62 @@ class IMAPFolder(BaseFolder): try: # Primes untagged_responses imapobj.select(self.getfullname(), readonly = 1, force = 1) - try: - # Some mail servers do not return an EXISTS response if - # the folder is empty. - maxmsgid = long(imapobj.untagged_responses['EXISTS'][0]) - except KeyError: - return - if maxmsgid < 1: - # No messages; return - return + maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) + maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) + + if (maxage != -1) | (maxsize != -1): + try: + search_condition = "("; + + if(maxage != -1): + #find out what the oldest message is that we should look at + oldest_time_struct = time.gmtime(time.time() - (60*60*24*maxage)) + + #format this manually - otherwise locales could cause problems + monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \ + "June", "July", "Aug", "Sep", "Oct", "Nov", "Dec"] + + our_monthname = monthnames_standard[oldest_time_struct[1]-1] + daystr = "%(day)02d" % {'day' : oldest_time_struct[2]} + date_search_str = "SINCE " + daystr + "-" + our_monthname \ + + "-" + str(oldest_time_struct[0]) + + search_condition += date_search_str + + if(maxsize != -1): + if(maxage != 1): #There are two conditions - add a space + search_condition += " " + + search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1) + + search_condition += ")" + searchresult = imapobj.search(None, search_condition) + + #result would come back seperated by space - to change into a fetch + #statement we need to change space to comma + messagesToFetch = searchresult[1][0].replace(" ", ",") + except KeyError: + return + if len(messagesToFetch) < 1: + # No messages; return + return + else: + try: + # Some mail servers do not return an EXISTS response if + # the folder is empty. + + maxmsgid = long(imapobj.untagged_responses['EXISTS'][0]) + messagesToFetch = '1:%d' % maxmsgid; + except KeyError: + return + if maxmsgid < 1: + #no messages; return + return # Now, get the flags and UIDs for these. # We could conceivably get rid of maxmsgid and just say # '1:*' here. - response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1] + response = imapobj.fetch(messagesToFetch, '(FLAGS UID)')[1] finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 811d759..49d4dae 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -29,6 +29,7 @@ except ImportError: uidmatchre = re.compile(',U=(\d+)') flagmatchre = re.compile(':.*2,([A-Z]+)') +timestampmatchre = re.compile('(\d+)'); timeseq = 0 lasttime = long(0) @@ -72,6 +73,28 @@ class MaildirFolder(BaseFolder): token.""" return 42 + #Checks to see if the given message is within the maximum age according + #to the maildir name which should begin with a timestamp + def _iswithinmaxage(self, messagename, maxage): + #In order to have the same behaviour as SINCE in an IMAP search + #we must convert this to the oldest time and then strip off hrs/mins + #from that day + oldest_time_utc = time.time() - (60*60*24*maxage) + oldest_time_struct = time.gmtime(oldest_time_utc) + oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \ + + (oldest_time_struct[4] * 60) \ + + oldest_time_struct[5]) + oldest_time_utc -= oldest_time_today_seconds + + timestampmatch = timestampmatchre.search(messagename) + timestampstr = timestampmatch.group() + timestamplong = long(timestampstr) + if(timestamplong < oldest_time_utc): + return False + else: + return True + + def _scanfolder(self): """Cache the message list. Maildir flags are: R (replied) @@ -92,6 +115,25 @@ class MaildirFolder(BaseFolder): filename in os.listdir(fulldirname)) for file in files: messagename = os.path.basename(file) + + #check if there is a parameter for maxage / maxsize - then see if this + #message should be considered or not + maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) + maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) + + if(maxage != -1): + isnewenough = self._iswithinmaxage(messagename, maxage) + if(isnewenough != True): + #this message is older than we should consider.... + continue + + #Check and see if the message is too big if the maxsize for this account is set + if(maxsize != -1): + filesize = os.path.getsize(file) + if(filesize > maxsize): + continue + + foldermatch = messagename.find(folderstr) != -1 if not foldermatch: # If there is no folder MD5 specified, or if it mismatches,