mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix local video path cache (hopefully)

Browse Source
This commit is contained in:
I-Al-Istannen 2022-01-08 00:27:48 +01:00
parent a99356f2a2
commit 462d993fbc
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -489,7 +489,10 @@ instance's greatest bottleneck.
log.explain_topic(f"Checking local cache for video {video_path.name}") log.explain_topic(f"Checking local cache for video {video_path.name}")
all_found_locally = True all_found_locally = True
for video in contained_videos: for video in contained_videos:
all_found_locally = all_found_locally and self._output_dir.resolve(video).exists() transformed_path = self._transformer.transform(video)
if transformed_path:
exists_locally = self._output_dir.resolve(transformed_path).exists()
all_found_locally = all_found_locally and exists_locally
if all_found_locally: if all_found_locally:
log.explain("Found all videos locally, skipping enumeration request") log.explain("Found all videos locally, skipping enumeration request")
return True return True
@ -515,8 +518,12 @@ instance's greatest bottleneck.
log.explain(f"Using single video mode for {element.name}") log.explain(f"Using single video mode for {element.name}")
stream_element = stream_elements[0] stream_element = stream_elements[0]
transformed_path = self._transformer.transform(original_path)
if not transformed_path:
raise CrawlError(f"Download returned a path but transform did not for {original_path}")
# We do not have a local cache yet # We do not have a local cache yet
if self._output_dir.resolve(original_path).exists(): if self._output_dir.resolve(transformed_path).exists():
log.explain(f"Video for {element.name} existed locally") log.explain(f"Video for {element.name} existed locally")
else: else:
await self._stream_from_url(stream_element.url, sink, bar, is_video=True) await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
@ -526,8 +533,8 @@ instance's greatest bottleneck.
contained_video_paths: List[str] = [] contained_video_paths: List[str] = []
for stream_element in stream_elements: for stream_element in stream_elements:
contained_video_paths.append(stream_element.name)
video_path = original_path.parent / stream_element.name video_path = original_path.parent / stream_element.name
contained_video_paths.append(str(video_path))
maybe_dl = await self.download(video_path, mtime=element.mtime, redownload=Redownload.NEVER) maybe_dl = await self.download(video_path, mtime=element.mtime, redownload=Redownload.NEVER)
if not maybe_dl: if not maybe_dl: