mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix renaming conflict with multi-stage video elements

Browse Source
This commit is contained in:
I-Al-Istannen 2021-05-27 00:55:46 +02:00 committed by Joscha
parent 19eed5bdff
commit 5beb4d9a2d
2 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
PFERD/crawl/ilias/kit_ilias_html.py
View File

@ -62,14 +62,16 @@ class IliasPage:
log.explain("Page is a normal folder, searching for elements") log.explain("Page is a normal folder, searching for elements")
return self._find_normal_entries() return self._find_normal_entries()
def get_next_stage_url(self) -> Optional[str]:
if self._is_ilias_opencast_embedding():
return self.get_child_elements()[0].url
return None
def _is_video_player(self) -> bool: def _is_video_player(self) -> bool:
return "paella_config_file" in str(self._soup) return "paella_config_file" in str(self._soup)
def _is_video_listing(self) -> bool: def _is_video_listing(self) -> bool:
# ILIAS fluff around it if self._is_ilias_opencast_embedding():
if self._soup.find(id="headerimage"):
element: Tag = self._soup.find(id="headerimage")
if "opencast" in element.attrs["src"].lower():
return True return True
# Raw listing without ILIAS fluff # Raw listing without ILIAS fluff
@ -78,6 +80,14 @@ class IliasPage:
) )
return video_element_table is not None return video_element_table is not None
def _is_ilias_opencast_embedding(self) -> bool:
# ILIAS fluff around the real opencast html
if self._soup.find(id="headerimage"):
element: Tag = self._soup.find(id="headerimage")
if "opencast" in element.attrs["src"].lower():
return True
return False
def _is_exercise_file(self) -> bool: def _is_exercise_file(self) -> bool:
# we know it from before # we know it from before
if self._page_type == IliasElementType.EXERCISE: if self._page_type == IliasElementType.EXERCISE:

6
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -242,10 +242,14 @@ class KitIliasWebCrawler(HttpCrawler):
async def gather_elements() -> None: async def gather_elements() -> None:
elements.clear() elements.clear()
async with cl: async with cl:
soup = await self._get_page(url) next_stage_url: Optional[str] = url
log.explain_topic(f"Parsing HTML page for {fmt_path(path)}") log.explain_topic(f"Parsing HTML page for {fmt_path(path)}")
while next_stage_url:
soup = await self._get_page(next_stage_url)
log.explain(f"URL: {url}") log.explain(f"URL: {url}")
page = IliasPage(soup, url, parent) page = IliasPage(soup, url, parent)
next_stage_url = page.get_next_stage_url()
elements.extend(page.get_child_elements()) elements.extend(page.get_child_elements())