Merge branch 'master' of ssh://jpgarch@complete.org/~jpgarch/git/offlineimap

debian/changelog

@@ -1,7 +1,7 @@
 offlineimap (6.2.0) unstable; urgency=low
 
   * FIX: Removed imaplib2 entirely.  It was the cause of numerous
-    crash/hang bugs.
+    crash/hang bugs.  Closes: #536156, #536329.
   * REMOVAL: IDLE support had to be removed because it depended on
     imaplib2.
   * Do not request INTERNALDATE when it is not needed.

offlineimap.conf

@@ -195,6 +195,30 @@ remoterepository = RemoteExample
 # You can also specify parameters to the commands
 # presynchook = imapfilter -c someotherconfig.lua
 
+# If you have a limited amount of bandwidth available you can exclude larger
+# messages (e.g. those with large attachments etc).  If you do this it
+# will appear to offlineimap that these messages do not exist at all.  They
+# will not be copied, have flags changed etc.  For this to work on an IMAP
+# server the server must have server side search enabled.  This works with gmail
+# and most imap servers (e.g. cyrus etc)
+# The maximum size should be specified in bytes - e.g. 2000000 for approx 2MB
+
+# maxsize = 2000000
+
+
+# When you are starting to sync an already existing account yuo can tell offlineimap
+# to sync messages from only the last x days.  When you do this messages older than x
+# days will be completely ignored.  This can be useful for importing existing accounts
+# when you do not want to download large amounts of archive email.
+
+# Messages older than maxage days will not be synced, their flags will
+# not be changed, they will not be deleted etc.  For offlineimap it will be like these
+# messages do not exist.  This will perform an IMAP search in the case of IMAP or Gmail
+# and therefor requires that the server support server side searching.  This will
+# calculate the earliest day that would be included in the search and include all 
+# messages from that day until today.   e.g. maxage = 3 to sync only the last 3 days mail
+
+# maxage = 3
 
 [Repository LocalExample]
 

offlineimap/accounts.py

@@ -178,16 +178,22 @@ class AccountSynchronizationMixin:
 
         # Connect to the local cache.
         self.statusrepos = offlineimap.repository.LocalStatus.LocalStatusRepository(self.getconf('localrepository'), self)
-            
+
+        #might need changes here to ensure that one account sync does not crash others...
         if not self.refreshperiod:
+            
             self.sync(siglistener)
             self.ui.acctdone(self.name)
+
             return
+
+
         looping = 1
         while looping:
             self.sync(siglistener)
             looping = self.sleeper(siglistener) != 2
-        self.ui.acctdone(self.name)
+            self.ui.acctdone(self.name)
+
 
     def getaccountmeta(self):
         return os.path.join(self.metadatadir, 'Account-' + self.name)

offlineimap/folder/IMAP.py

@@ -24,6 +24,7 @@ from offlineimap.version import versionstr
 import rfc822, time, string, random, binascii, re
 from StringIO import StringIO
 from copy import copy
+import time
 
 
 class IMAPFolder(BaseFolder):
@@ -116,6 +117,7 @@ class IMAPFolder(BaseFolder):
 
         return False
 
+    # TODO: Make this so that it can define a date that would be the oldest messages etc.
     def cachemessagelist(self):
         imapobj = self.imapserver.acquireconnection()
         self.messagelist = {}
@@ -123,20 +125,62 @@ class IMAPFolder(BaseFolder):
         try:
             # Primes untagged_responses
             imapobj.select(self.getfullname(), readonly = 1, force = 1)
-            try:
-                # Some mail servers do not return an EXISTS response if
-                # the folder is empty.
-                maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
-            except KeyError:
-                return
-            if maxmsgid < 1:
-                # No messages; return
-                return
 
+            maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1)
+            maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1)
+
+            if (maxage != -1) | (maxsize != -1):
+                try:
+                    search_condition = "(";
+
+                    if(maxage != -1):
+                        #find out what the oldest message is that we should look at
+                        oldest_time_struct = time.gmtime(time.time() - (60*60*24*maxage))
+
+                        #format this manually - otherwise locales could cause problems
+                        monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \
+                            "June", "July", "Aug", "Sep", "Oct", "Nov", "Dec"]
+
+                        our_monthname = monthnames_standard[oldest_time_struct[1]-1]
+                        daystr = "%(day)02d" % {'day' : oldest_time_struct[2]}
+                        date_search_str = "SINCE " + daystr + "-" + our_monthname \
+                            + "-" + str(oldest_time_struct[0])
+
+                        search_condition += date_search_str
+
+                    if(maxsize != -1):
+                        if(maxage != 1): #There are two conditions - add a space
+                            search_condition += " "
+
+                        search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1)
+
+                    search_condition += ")"
+                    searchresult = imapobj.search(None, search_condition)
+
+                    #result would come back seperated by space - to change into a fetch
+                    #statement we need to change space to comma
+                    messagesToFetch = searchresult[1][0].replace(" ", ",")
+                except KeyError:
+                    return
+                if len(messagesToFetch) < 1:
+                    # No messages; return
+                    return
+            else:
+                try:
+                    # Some mail servers do not return an EXISTS response if
+                    # the folder is empty.
+
+                    maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
+                    messagesToFetch = '1:%d' % maxmsgid;
+                except KeyError:
+                    return
+                if maxmsgid < 1:
+                    #no messages; return
+                    return
             # Now, get the flags and UIDs for these.
             # We could conceivably get rid of maxmsgid and just say
             # '1:*' here.
-            response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
+            response = imapobj.fetch(messagesToFetch, '(FLAGS UID)')[1]
         finally:
             self.imapserver.releaseconnection(imapobj)
         for messagestr in response:

offlineimap/folder/Maildir.py

@@ -29,6 +29,7 @@ except ImportError:
 
 uidmatchre = re.compile(',U=(\d+)')
 flagmatchre = re.compile(':.*2,([A-Z]+)')
+timestampmatchre = re.compile('(\d+)');
 
 timeseq = 0
 lasttime = long(0)
@@ -72,6 +73,28 @@ class MaildirFolder(BaseFolder):
         token."""
         return 42
 
+    #Checks to see if the given message is within the maximum age according
+    #to the maildir name which should begin with a timestamp
+    def _iswithinmaxage(self, messagename, maxage):
+        #In order to have the same behaviour as SINCE in an IMAP search
+        #we must convert this to the oldest time and then strip off hrs/mins
+        #from that day
+        oldest_time_utc = time.time() - (60*60*24*maxage)
+        oldest_time_struct = time.gmtime(oldest_time_utc)
+        oldest_time_today_seconds = ((oldest_time_struct[3] * 3600) \
+            + (oldest_time_struct[4] * 60) \
+            + oldest_time_struct[5])
+        oldest_time_utc -= oldest_time_today_seconds
+
+        timestampmatch = timestampmatchre.search(messagename)
+        timestampstr = timestampmatch.group()
+        timestamplong = long(timestampstr)
+        if(timestamplong < oldest_time_utc):
+            return False
+        else:
+            return True
+
+
     def _scanfolder(self):
         """Cache the message list.  Maildir flags are:
         R (replied)
@@ -92,6 +115,25 @@ class MaildirFolder(BaseFolder):
                          filename in os.listdir(fulldirname))
         for file in files:
             messagename = os.path.basename(file)
+
+            #check if there is a parameter for maxage / maxsize - then see if this
+            #message should be considered or not
+            maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1)
+            maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1)
+
+            if(maxage != -1):
+                isnewenough = self._iswithinmaxage(messagename, maxage)
+                if(isnewenough != True):
+                    #this message is older than we should consider....
+                    continue
+
+            #Check and see if the message is too big if the maxsize for this account is set
+            if(maxsize != -1):
+                filesize = os.path.getsize(file)
+                if(filesize > maxsize):
+                    continue
+            
+
             foldermatch = messagename.find(folderstr) != -1
             if not foldermatch:
                 # If there is no folder MD5 specified, or if it mismatches,