mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use fixed windows path for video cache

Browse Source
This commit is contained in:
I-Al-Istannen 2022-01-14 22:01:45 +01:00
parent 57ec51e95a
commit f47e7374d2
3 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -23,8 +23,10 @@ ambiguous situations.
## Unreleased ## Unreleased
### Fixed ### Fixed
- Shibboleth login fixed. It was broken due to URL parser changes and really - Shibboleth login. It was broken due to URL parser changes and really
*unfortunate* behaviour by aiohttp. *unfortunate* behaviour by aiohttp.
- local video cache on windows if the path was changed to accomodate windows
file system limitations (e.g. replace `:`)
## 3.3.0 - 2022-01-09 ## 3.3.0 - 2022-01-09

9
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -499,7 +499,7 @@ instance's greatest bottleneck.
log.explain_topic(f"Checking local cache for video {video_path.name}") log.explain_topic(f"Checking local cache for video {video_path.name}")
all_found_locally = True all_found_locally = True
for video in contained_videos: for video in contained_videos:
transformed_path = self._transformer.transform(video) transformed_path = self._to_local_video_path(video)
if transformed_path: if transformed_path:
exists_locally = self._output_dir.resolve(transformed_path).exists() exists_locally = self._output_dir.resolve(transformed_path).exists()
all_found_locally = all_found_locally and exists_locally all_found_locally = all_found_locally and exists_locally
@ -509,6 +509,11 @@ instance's greatest bottleneck.
log.explain("Missing at least one video, continuing with requests!") log.explain("Missing at least one video, continuing with requests!")
return False return False
def _to_local_video_path(self, path: PurePath) -> Optional[PurePath]:
if transformed := self._transformer.transform(path):
return self._deduplicator.fixup_path(transformed)
return None
@anoncritical @anoncritical
@_iorepeat(3, "downloading video") @_iorepeat(3, "downloading video")
async def _download_video( async def _download_video(
@ -528,7 +533,7 @@ instance's greatest bottleneck.
log.explain(f"Using single video mode for {element.name}") log.explain(f"Using single video mode for {element.name}")
stream_element = stream_elements[0] stream_element = stream_elements[0]
transformed_path = self._transformer.transform(original_path) transformed_path = self._to_local_video_path(original_path)
if not transformed_path: if not transformed_path:
raise CrawlError(f"Download returned a path but transform did not for {original_path}") raise CrawlError(f"Download returned a path but transform did not for {original_path}")

6
PFERD/deduplicator.py
View File

@ -56,6 +56,12 @@ class Deduplicator:
log.explain(f"Changed path to {fmt_path(new_path)} for windows compatibility") log.explain(f"Changed path to {fmt_path(new_path)} for windows compatibility")
return new_path return new_path
def fixup_path(self, path: PurePath) -> PurePath:
"""Fixes up the path for windows, if enabled. Returns the path unchanged otherwise."""
if self._windows_paths:
return self._fixup_for_windows(path)
return path
def mark(self, path: PurePath) -> PurePath: def mark(self, path: PurePath) -> PurePath:
if self._windows_paths: if self._windows_paths:
path = self._fixup_for_windows(path) path = self._fixup_for_windows(path)