mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Handle empty forums

Browse Source
This commit is contained in:
I-Al-Istannen
2022-10-24 13:09:29 +02:00
parent 4a51aaa4f5
commit d72fc2760b
3 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
PFERD/crawl/ilias/kit_ilias_web_crawler.py
View File

@ -658,7 +658,7 @@ instance's greatest bottleneck.
@_iorepeat(3, "crawling forum")
@anoncritical
async def _crawl_forum(self, element: IliasPageElement, cl: CrawlToken) -> None:
elements = []
elements: List[IliasForumThread] = []
async with cl:
next_stage_url = element.url
@ -677,6 +677,10 @@ instance's greatest bottleneck.
download_data = page.get_download_forum_data()
if not download_data:
raise CrawlWarning("Failed to extract forum data")
if download_data.empty:
log.explain("Forum had no threads")
elements = []
return
html = await self._post_authenticated(download_data.url, download_data.form_data)
elements = parse_ilias_forum_export(soupify(html))