mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix authentication logic conflicts with videos

Browse Source
This commit is contained in:
I-Al-Istannen 2021-05-27 00:31:36 +02:00 committed by Joscha
parent 6fa9cfd4c3
commit 19eed5bdff
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -360,7 +360,9 @@ class KitIliasWebCrawler(HttpCrawler):
page = IliasPage(await self._get_page(element.url), element.url, element) page = IliasPage(await self._get_page(element.url), element.url, element)
real_element = page.get_child_elements()[0] real_element = page.get_child_elements()[0]
await self._stream_from_url(real_element.url, sink, bar) log.explain(f"Streaming video from real url {real_element.url}")
await self._stream_from_url(real_element.url, sink, bar, is_video=True)
await impl() await impl()
@ -374,16 +376,20 @@ class KitIliasWebCrawler(HttpCrawler):
async def impl() -> None: async def impl() -> None:
assert dl # The function is only reached when dl is not None assert dl # The function is only reached when dl is not None
async with dl as (bar, sink): async with dl as (bar, sink):
await self._stream_from_url(element.url, sink, bar) await self._stream_from_url(element.url, sink, bar, is_video=False)
await impl() await impl()
async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar) -> None: async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar, is_video: bool) -> None:
async def try_stream() -> bool: async def try_stream() -> bool:
async with self.session.get(url, allow_redirects=False) as resp: async with self.session.get(url, allow_redirects=is_video) as resp:
if not is_video:
# Redirect means we weren't authenticated # Redirect means we weren't authenticated
if hdrs.LOCATION in resp.headers: if hdrs.LOCATION in resp.headers:
return False return False
# we wanted a video but got HTML
if is_video and "html" in resp.content_type:
return False
if resp.content_length: if resp.content_length:
bar.set_total(resp.content_length) bar.set_total(resp.content_length)