From 1cfd3fd95b11c1b20e167ec288eb5e11ee3f6d29 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 17 Sep 2016 16:53:24 +0200 Subject: [PATCH 1/6] filter broken vcard entries before delivered, found on real life system at least 2 cards where vobject claimed about vobject.base.ValidateError: 'VCARD components must contain at least 1 N' --- radicale/storage.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/radicale/storage.py b/radicale/storage.py index a67fbf9..d82c736 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -718,7 +718,18 @@ class Collection(BaseCollection): break return collection.serialize() elif self.get_meta("tag") == "VADDRESSBOOK": - return "".join([item.serialize() for item in items]) + self.logger.debug("Try to serialize VADDRESSBOOK") + items_ok = [] + result = "" + for item in items: + try: + item.serialize() + items_ok.append(item) + except: + self.logger.exception("VCARD broken (skip): %s", item) + result = "".join([item.serialize() for item in items_ok]) + self.logger.debug("Try to return VADDRESSBOOK") + return result return "" _lock = threading.Lock() From ff02d17619d2afaf43e20c812c5b140db85e8e1c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 17 Sep 2016 17:44:33 +0200 Subject: [PATCH 2/6] remove temporary debug statements --- radicale/storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/radicale/storage.py b/radicale/storage.py index d82c736..3edae58 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -718,7 +718,6 @@ class Collection(BaseCollection): break return collection.serialize() elif self.get_meta("tag") == "VADDRESSBOOK": - self.logger.debug("Try to serialize VADDRESSBOOK") items_ok = [] result = "" for item in items: @@ -728,7 +727,6 @@ class Collection(BaseCollection): except: self.logger.exception("VCARD broken (skip): %s", item) result = "".join([item.serialize() for item in items_ok]) - self.logger.debug("Try to return VADDRESSBOOK") return result return "" From 1ffdb0310676e296144a3f64738d52abf579878c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 17 Sep 2016 18:16:27 +0200 Subject: [PATCH 3/6] simplify patch --- radicale/storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/radicale/storage.py b/radicale/storage.py index 3edae58..6a27d3a 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -719,15 +719,13 @@ class Collection(BaseCollection): return collection.serialize() elif self.get_meta("tag") == "VADDRESSBOOK": items_ok = [] - result = "" for item in items: try: item.serialize() items_ok.append(item) except: self.logger.exception("VCARD broken (skip): %s", item) - result = "".join([item.serialize() for item in items_ok]) - return result + return "".join([item.serialize() for item in items_ok]) return "" _lock = threading.Lock() From af5c1582dcf35794a5fed0c0e93978ff49f54293 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 18 Sep 2016 19:23:24 +0200 Subject: [PATCH 4/6] improved detection of broken vcards --- radicale/storage.py | 14 +++++++++++++- radicale/xmlutils.py | 14 +++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/radicale/storage.py b/radicale/storage.py index b42b862..0f276d9 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -715,9 +715,21 @@ class Collection(BaseCollection): self.logger.debug("Read object: %s", path) with open(path, encoding=self.encoding, newline="") as fd: try: - items.append(vobject.readOne(fd.read())) + # check whether vobject liks the item + item = vobject.readOne(fd.read()) except: self.logger.exception("Object broken (skip 'list'): %s", path) + continue + + if self.get_meta("tag") == "VADDRESSBOOK": + try: + # check whether vobject liks the VCARD item + item.serialize() + except: + self.logger.exception("Object broken (skip 'read'): %s", path) + self.logger.error("Broken VCARD content: %s", item) + continue + items.append(item) time_end = datetime.datetime.now() self.logger.info("Collection read %d items in %s sec from %s", len(items),(time_end - time_begin).total_seconds(), self._filesystem_path) if self.get_meta("tag") == "VCALENDAR": diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index 1c2a428..37bad2c 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -28,6 +28,7 @@ in them for XML requests (all but PUT). import posixpath import re import xml.etree.ElementTree as ET +from pprint import pprint from collections import OrderedDict from datetime import datetime, timedelta, timezone from http import client @@ -523,13 +524,15 @@ def propfind(path, xml_request, read_collections, write_collections, user): collections.append(collection) response = _propfind_response( path, collection, props, user, write=True) - multistatus.append(response) + if response: + multistatus.append(response) for collection in read_collections: if collection in collections: continue response = _propfind_response( path, collection, props, user, write=False) - multistatus.append(response) + if response: + multistatus.append(response) return client.MULTI_STATUS, _pretty_xml(multistatus) @@ -569,7 +572,12 @@ def _propfind_response(path, item, props, user, write=False): element = ET.Element(tag) is404 = False if tag == _tag("D", "getetag"): - element.text = item.etag + try: + element.text = item.etag + except: + print("Object broken (skip)") + pprint(vars(item)) + return None elif tag == _tag("D", "getlastmodified"): element.text = item.last_modified elif tag == _tag("D", "principal-collection-set"): From a5e3ffa4270de4ac3d96929a152695012a291715 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 18 Sep 2016 19:30:51 +0200 Subject: [PATCH 5/6] realign change --- radicale/storage.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/radicale/storage.py b/radicale/storage.py index ea4f1ce..0f276d9 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -742,14 +742,7 @@ class Collection(BaseCollection): break return collection.serialize() elif self.get_meta("tag") == "VADDRESSBOOK": - items_ok = [] - for item in items: - try: - item.serialize() - items_ok.append(item) - except: - self.logger.exception("VCARD broken (skip): %s", item) - return "".join([item.serialize() for item in items_ok]) + return "".join([item.serialize() for item in items]) return "" _lock = threading.Lock() From 830d6f93056264b123ba0f427c1436178af0ddaa Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 18 Sep 2016 19:36:54 +0200 Subject: [PATCH 6/6] add examples of broken vcard and vevent --- radicale/tests/static/broken-vcard.vcf | 8 ++++++++ radicale/tests/static/broken-vevent.ics | 15 +++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 radicale/tests/static/broken-vcard.vcf create mode 100644 radicale/tests/static/broken-vevent.ics diff --git a/radicale/tests/static/broken-vcard.vcf b/radicale/tests/static/broken-vcard.vcf new file mode 100644 index 0000000..140ddc2 --- /dev/null +++ b/radicale/tests/static/broken-vcard.vcf @@ -0,0 +1,8 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000000.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example.com +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000000.vcf +END:VCARD diff --git a/radicale/tests/static/broken-vevent.ics b/radicale/tests/static/broken-vevent.ics new file mode 100644 index 0000000..a6244ea --- /dev/null +++ b/radicale/tests/static/broken-vevent.ics @@ -0,0 +1,15 @@ +BEGIN:VCALENDAR +PRODID:-//Radicale//NONSGML Radicale Server//EN +VERSION:2.0 +BEGIN:VEVENT +CREATED:20160725T060147Z +LAST-MODIFIED:20160727T193435Z +DTSTAMP:20160727T193435Z +UID:040000008200E00074C5B7101A82E00800000000 +SUMMARY:Broken ICS END of VEVENT missing by accident +STATUS:CONFIRMED +X-MOZ-LASTACK:20160727T193435Z +DTSTART;TZID=Europe/Budapest:20160727T170000 +DTEND;TZID=Europe/Budapest:20160727T223000 +CLASS:PUBLIC +X-LIC-ERROR:No value for LOCATION property. Removing entire property: