mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Retry crawling on I/O failure

Browse Source
This commit is contained in:
I-Al-Istannen 2021-05-15 22:46:26 +02:00
parent 9fd356d290
commit cf6903d109
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
PFERD/crawlers/ilias.py
View File

@ -596,12 +596,15 @@ class IliasCrawler(HttpCrawler):
# TODO: Proper exception # TODO: Proper exception
raise RuntimeError("Get page failed too often") raise RuntimeError("Get page failed too often")
print(url) print(url)
try:
async with self.session.get(url) as request: async with self.session.get(url) as request:
soup = soupify(await request.read()) soup = soupify(await request.read())
if self._is_logged_in(soup): if self._is_logged_in(soup):
return soup return soup
await self._shibboleth_login.login(self.session) await self._shibboleth_login.login(self.session)
except Exception:
return await self._get_page(url, retries_left - 1)
return await self._get_page(url, retries_left - 1) return await self._get_page(url, retries_left - 1)