mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Document authentication in HTTP crawler and rename prepare_request

Browse Source
This commit is contained in:
I-Al-Istannen
2021-05-23 11:55:34 +02:00
parent 25e2abdb03
commit 33a81a5f5c
2 changed files with 20 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
PFERD/crawlers/ilias/kit_ilias_web_crawler.py
View File

@ -333,7 +333,7 @@ class KitIliasWebCrawler(HttpCrawler):
sink.done()
return True
auth_id = await self.prepare_request()
auth_id = await self._current_auth_id()
if await try_stream():
return
@ -343,7 +343,7 @@ class KitIliasWebCrawler(HttpCrawler):
raise CrawlError("File streaming failed after authenticate()")
async def _get_page(self, url: str) -> BeautifulSoup:
auth_id = await self.prepare_request()
auth_id = await self._current_auth_id()
async with self.session.get(url) as request:
soup = soupify(await request.read())
if self._is_logged_in(soup):