From f47e7374d23b71396b511ee7b57f59d46c34e00d Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Fri, 14 Jan 2022 22:01:45 +0100 Subject: [PATCH] Use fixed windows path for video cache --- CHANGELOG.md | 4 +++- PFERD/crawl/ilias/kit_ilias_web_crawler.py | 9 +++++++-- PFERD/deduplicator.py | 6 ++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41ee3d5..7f35a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,10 @@ ambiguous situations. ## Unreleased ### Fixed -- Shibboleth login fixed. It was broken due to URL parser changes and really +- Shibboleth login. It was broken due to URL parser changes and really *unfortunate* behaviour by aiohttp. +- local video cache on windows if the path was changed to accomodate windows + file system limitations (e.g. replace `:`) ## 3.3.0 - 2022-01-09 diff --git a/PFERD/crawl/ilias/kit_ilias_web_crawler.py b/PFERD/crawl/ilias/kit_ilias_web_crawler.py index c26ce8b..b197b6b 100644 --- a/PFERD/crawl/ilias/kit_ilias_web_crawler.py +++ b/PFERD/crawl/ilias/kit_ilias_web_crawler.py @@ -499,7 +499,7 @@ instance's greatest bottleneck. log.explain_topic(f"Checking local cache for video {video_path.name}") all_found_locally = True for video in contained_videos: - transformed_path = self._transformer.transform(video) + transformed_path = self._to_local_video_path(video) if transformed_path: exists_locally = self._output_dir.resolve(transformed_path).exists() all_found_locally = all_found_locally and exists_locally @@ -509,6 +509,11 @@ instance's greatest bottleneck. log.explain("Missing at least one video, continuing with requests!") return False + def _to_local_video_path(self, path: PurePath) -> Optional[PurePath]: + if transformed := self._transformer.transform(path): + return self._deduplicator.fixup_path(transformed) + return None + @anoncritical @_iorepeat(3, "downloading video") async def _download_video( @@ -528,7 +533,7 @@ instance's greatest bottleneck. log.explain(f"Using single video mode for {element.name}") stream_element = stream_elements[0] - transformed_path = self._transformer.transform(original_path) + transformed_path = self._to_local_video_path(original_path) if not transformed_path: raise CrawlError(f"Download returned a path but transform did not for {original_path}") diff --git a/PFERD/deduplicator.py b/PFERD/deduplicator.py index ef62dcb..7777f28 100644 --- a/PFERD/deduplicator.py +++ b/PFERD/deduplicator.py @@ -56,6 +56,12 @@ class Deduplicator: log.explain(f"Changed path to {fmt_path(new_path)} for windows compatibility") return new_path + def fixup_path(self, path: PurePath) -> PurePath: + """Fixes up the path for windows, if enabled. Returns the path unchanged otherwise.""" + if self._windows_paths: + return self._fixup_for_windows(path) + return path + def mark(self, path: PurePath) -> PurePath: if self._windows_paths: path = self._fixup_for_windows(path)