mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Only call should_crawl once

Browse Source
This commit is contained in:
I-Al-Istannen 2021-05-19 21:57:55 +02:00
parent 81301f3a76
commit 8cfa818f04
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
PFERD/crawlers/ilias/kit_ilias_web_crawler.py
View File

@ -130,6 +130,12 @@ class KitIliasWebCrawler(HttpCrawler):
@arepeat(3) @arepeat(3)
@anoncritical @anoncritical
async def _handle_ilias_page(self, url: str, parent: IliasPageElement, path: PurePath) -> None: async def _handle_ilias_page(self, url: str, parent: IliasPageElement, path: PurePath) -> None:
# We might not want to crawl this directory-ish page.
# This is not in #handle_element, as the download methods check it themselves and therefore
# would perform this check twice - messing with the explain output
if not self.should_crawl(path):
return
tasks = [] tasks = []
async with self.crawl_bar(path): async with self.crawl_bar(path):
soup = await self._get_page(url) soup = await self._get_page(url)