From 90565612119d4a0e128436a788839cd585dfc80c Mon Sep 17 00:00:00 2001 From: Florian Mounier Date: Thu, 7 Jul 2016 14:30:10 +0200 Subject: [PATCH 1/4] Move test inside radicale --- pytest.ini | 2 +- {tests => radicale/tests}/__init__.py | 0 {tests => radicale/tests}/custom/__init__.py | 0 {tests => radicale/tests}/custom/auth.py | 0 {tests => radicale/tests}/custom/storage.py | 0 {tests => radicale/tests}/helpers.py | 0 {tests => radicale/tests}/static/event1.ics | 0 {tests => radicale/tests}/static/event2.ics | 0 {tests => radicale/tests}/static/event3.ics | 0 {tests => radicale/tests}/static/event4.ics | 0 {tests => radicale/tests}/static/event5.ics | 0 {tests => radicale/tests}/static/journal1.ics | 0 {tests => radicale/tests}/static/journal2.ics | 0 {tests => radicale/tests}/static/journal3.ics | 0 {tests => radicale/tests}/static/journal4.ics | 0 {tests => radicale/tests}/static/journal5.ics | 0 {tests => radicale/tests}/static/todo1.ics | 0 {tests => radicale/tests}/static/todo2.ics | 0 {tests => radicale/tests}/static/todo3.ics | 0 {tests => radicale/tests}/static/todo4.ics | 0 {tests => radicale/tests}/static/todo5.ics | 0 {tests => radicale/tests}/static/todo6.ics | 0 {tests => radicale/tests}/static/todo7.ics | 0 {tests => radicale/tests}/static/todo8.ics | 0 {tests => radicale/tests}/test_auth.py | 0 {tests => radicale/tests}/test_base.py | 0 26 files changed, 1 insertion(+), 1 deletion(-) rename {tests => radicale/tests}/__init__.py (100%) rename {tests => radicale/tests}/custom/__init__.py (100%) rename {tests => radicale/tests}/custom/auth.py (100%) rename {tests => radicale/tests}/custom/storage.py (100%) rename {tests => radicale/tests}/helpers.py (100%) rename {tests => radicale/tests}/static/event1.ics (100%) rename {tests => radicale/tests}/static/event2.ics (100%) rename {tests => radicale/tests}/static/event3.ics (100%) rename {tests => radicale/tests}/static/event4.ics (100%) rename {tests => radicale/tests}/static/event5.ics (100%) rename {tests => radicale/tests}/static/journal1.ics (100%) rename {tests => radicale/tests}/static/journal2.ics (100%) rename {tests => radicale/tests}/static/journal3.ics (100%) rename {tests => radicale/tests}/static/journal4.ics (100%) rename {tests => radicale/tests}/static/journal5.ics (100%) rename {tests => radicale/tests}/static/todo1.ics (100%) rename {tests => radicale/tests}/static/todo2.ics (100%) rename {tests => radicale/tests}/static/todo3.ics (100%) rename {tests => radicale/tests}/static/todo4.ics (100%) rename {tests => radicale/tests}/static/todo5.ics (100%) rename {tests => radicale/tests}/static/todo6.ics (100%) rename {tests => radicale/tests}/static/todo7.ics (100%) rename {tests => radicale/tests}/static/todo8.ics (100%) rename {tests => radicale/tests}/test_auth.py (100%) rename {tests => radicale/tests}/test_base.py (100%) diff --git a/pytest.ini b/pytest.ini index da3dfa5..9bf10c8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = --flake8 --isort --cov radicale/ tests +addopts = --flake8 --isort --cov radicale/ radicale/tests norecursedirs = dist .cache .git build *.egg-info .eggs venv diff --git a/tests/__init__.py b/radicale/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to radicale/tests/__init__.py diff --git a/tests/custom/__init__.py b/radicale/tests/custom/__init__.py similarity index 100% rename from tests/custom/__init__.py rename to radicale/tests/custom/__init__.py diff --git a/tests/custom/auth.py b/radicale/tests/custom/auth.py similarity index 100% rename from tests/custom/auth.py rename to radicale/tests/custom/auth.py diff --git a/tests/custom/storage.py b/radicale/tests/custom/storage.py similarity index 100% rename from tests/custom/storage.py rename to radicale/tests/custom/storage.py diff --git a/tests/helpers.py b/radicale/tests/helpers.py similarity index 100% rename from tests/helpers.py rename to radicale/tests/helpers.py diff --git a/tests/static/event1.ics b/radicale/tests/static/event1.ics similarity index 100% rename from tests/static/event1.ics rename to radicale/tests/static/event1.ics diff --git a/tests/static/event2.ics b/radicale/tests/static/event2.ics similarity index 100% rename from tests/static/event2.ics rename to radicale/tests/static/event2.ics diff --git a/tests/static/event3.ics b/radicale/tests/static/event3.ics similarity index 100% rename from tests/static/event3.ics rename to radicale/tests/static/event3.ics diff --git a/tests/static/event4.ics b/radicale/tests/static/event4.ics similarity index 100% rename from tests/static/event4.ics rename to radicale/tests/static/event4.ics diff --git a/tests/static/event5.ics b/radicale/tests/static/event5.ics similarity index 100% rename from tests/static/event5.ics rename to radicale/tests/static/event5.ics diff --git a/tests/static/journal1.ics b/radicale/tests/static/journal1.ics similarity index 100% rename from tests/static/journal1.ics rename to radicale/tests/static/journal1.ics diff --git a/tests/static/journal2.ics b/radicale/tests/static/journal2.ics similarity index 100% rename from tests/static/journal2.ics rename to radicale/tests/static/journal2.ics diff --git a/tests/static/journal3.ics b/radicale/tests/static/journal3.ics similarity index 100% rename from tests/static/journal3.ics rename to radicale/tests/static/journal3.ics diff --git a/tests/static/journal4.ics b/radicale/tests/static/journal4.ics similarity index 100% rename from tests/static/journal4.ics rename to radicale/tests/static/journal4.ics diff --git a/tests/static/journal5.ics b/radicale/tests/static/journal5.ics similarity index 100% rename from tests/static/journal5.ics rename to radicale/tests/static/journal5.ics diff --git a/tests/static/todo1.ics b/radicale/tests/static/todo1.ics similarity index 100% rename from tests/static/todo1.ics rename to radicale/tests/static/todo1.ics diff --git a/tests/static/todo2.ics b/radicale/tests/static/todo2.ics similarity index 100% rename from tests/static/todo2.ics rename to radicale/tests/static/todo2.ics diff --git a/tests/static/todo3.ics b/radicale/tests/static/todo3.ics similarity index 100% rename from tests/static/todo3.ics rename to radicale/tests/static/todo3.ics diff --git a/tests/static/todo4.ics b/radicale/tests/static/todo4.ics similarity index 100% rename from tests/static/todo4.ics rename to radicale/tests/static/todo4.ics diff --git a/tests/static/todo5.ics b/radicale/tests/static/todo5.ics similarity index 100% rename from tests/static/todo5.ics rename to radicale/tests/static/todo5.ics diff --git a/tests/static/todo6.ics b/radicale/tests/static/todo6.ics similarity index 100% rename from tests/static/todo6.ics rename to radicale/tests/static/todo6.ics diff --git a/tests/static/todo7.ics b/radicale/tests/static/todo7.ics similarity index 100% rename from tests/static/todo7.ics rename to radicale/tests/static/todo7.ics diff --git a/tests/static/todo8.ics b/radicale/tests/static/todo8.ics similarity index 100% rename from tests/static/todo8.ics rename to radicale/tests/static/todo8.ics diff --git a/tests/test_auth.py b/radicale/tests/test_auth.py similarity index 100% rename from tests/test_auth.py rename to radicale/tests/test_auth.py diff --git a/tests/test_base.py b/radicale/tests/test_base.py similarity index 100% rename from tests/test_base.py rename to radicale/tests/test_base.py From 364ed3689dcfc965a436bd560e6497cdeb474f8d Mon Sep 17 00:00:00 2001 From: Florian Mounier Date: Thu, 7 Jul 2016 15:45:01 +0200 Subject: [PATCH 2/4] Optimise collection creation by avoiding nested_loop --- radicale/storage.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/radicale/storage.py b/radicale/storage.py index 79d5edc..d670daf 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -35,7 +35,8 @@ import time from contextlib import contextmanager from hashlib import md5 from importlib import import_module -from uuid import uuid4 +from itertools import groupby +from random import getrandbits import vobject @@ -372,25 +373,26 @@ class Collection(BaseCollection): items = [] for content in ("vevent", "vtodo", "vjournal"): items.extend(getattr(collection, "%s_list" % content, [])) - processed_uids = [] - for i, item in enumerate(items): - uid = getattr(item, "uid", None) - if uid in processed_uids: - continue - new_collection = vobject.iCalendar() - new_collection.add(item) - if uid: - processed_uids.append(uid) - # search for items with same UID - for oitem in items[i+1:]: - if getattr(oitem, "uid", None) == uid: - new_collection.add(oitem) - self.upload(uuid4().hex, new_collection) + + def get_uid(item): + return hasattr(item, 'uid') and item.uid.value + + items_by_uid = groupby( + sorted(items, key=get_uid), get_uid) + + for uid, items in items_by_uid: + for item in items: + new_collection = vobject.iCalendar() + new_collection.add(item) + file_name = hex(getrandbits(32))[2:] + self.upload(file_name, new_collection) + elif tag == "VCARD": self.set_meta("tag", "VADDRESSBOOK") if collection: for card in collection: - self.upload(uuid4().hex, card) + file_name = hex(getrandbits(32))[2:] + self.upload(file_name, card) return self def list(self): From a2d6977cd6df4430558bf36dd6a3a71f51b3d9ca Mon Sep 17 00:00:00 2001 From: Florian Mounier Date: Thu, 7 Jul 2016 17:53:35 +0200 Subject: [PATCH 3/4] Fix logic in loops --- radicale/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/storage.py b/radicale/storage.py index a0cbe6d..443e496 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -381,8 +381,8 @@ class Collection(BaseCollection): sorted(items, key=get_uid), get_uid) for uid, items in items_by_uid: + new_collection = vobject.iCalendar() for item in items: - new_collection = vobject.iCalendar() new_collection.add(item) file_name = hex(getrandbits(32))[2:] self.upload(file_name, new_collection) From 39823f8909e8113fdbe34bd9bf82190bee570ad4 Mon Sep 17 00:00:00 2001 From: Florian Mounier Date: Fri, 8 Jul 2016 11:37:30 +0200 Subject: [PATCH 4/4] Add a pre_filtered_list method in collection. This allow filters optimizations --- radicale/storage.py | 9 +++++++++ radicale/xmlutils.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/radicale/storage.py b/radicale/storage.py index c50edf5..a54774f 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -231,6 +231,15 @@ class BaseCollection: for href in set(hrefs): yield self.get(href) + def pre_filtered_list(self, filters): + """List collection items with optional pre filtering. + + This could largely improve performance of reports depending on + the filters and this implementation. + This returns all event by default + """ + return [self.get(href) for href, _ in self.list()] + def has(self, href): """Check if an item exists by its href. diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index 71bb8a3..350dd1b 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -837,7 +837,7 @@ def report(path, xml_request, collection): else: # Reference is a collection path = hreference - items = [collection.get(href) for href, etag in collection.list()] + items = collection.pre_filtered_list(filters) for item in items: if filters: