mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Sort elements by ILIAS id to ensure deterministic ordering

Browse Source
This commit is contained in:
I-Al-Istannen 2021-07-06 17:45:12 +02:00
parent 8ec3f41251
commit ee67f9f472
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
PFERD/crawl/ilias/kit_ilias_html.py
View File

@ -38,6 +38,17 @@ class IliasPageElement:
mtime: Optional[datetime] = None mtime: Optional[datetime] = None
description: Optional[str] = None description: Optional[str] = None
def id(self) -> str:
regexes = [r"eid=(?P<id>[0-9a-z\-]+)", r"file_(?P<id>\d+)", r"ref_id=(?P<id>\d+)"]
for regex in regexes:
if match := re.search(regex, self.url):
return match.groupdict()["id"]
# Fall back to URL
log.warn(f"Didn't find identity for {self.name} - {self.url}. Please report this.")
return self.url
class IliasPage: class IliasPage:

4
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -230,6 +230,8 @@ instance's greatest bottleneck.
# Fill up our task list with the found elements # Fill up our task list with the found elements
await gather_elements() await gather_elements()
elements.sort(key=lambda e: e.id())
tasks: List[Awaitable[None]] = [] tasks: List[Awaitable[None]] = []
for element in elements: for element in elements:
if handle := await self._handle_ilias_element(PurePath("."), element): if handle := await self._handle_ilias_element(PurePath("."), element):
@ -280,6 +282,8 @@ instance's greatest bottleneck.
# Fill up our task list with the found elements # Fill up our task list with the found elements
await gather_elements() await gather_elements()
elements.sort(key=lambda e: e.id())
tasks: List[Awaitable[None]] = [] tasks: List[Awaitable[None]] = []
for element in elements: for element in elements:
if handle := await self._handle_ilias_element(cl.path, element): if handle := await self._handle_ilias_element(cl.path, element):