Retry crawling on I/O failure

This commit is contained in:
I-Al-Istannen 2021-05-15 22:46:26 +02:00
parent 9fd356d290
commit cf6903d109

View File

@ -596,12 +596,15 @@ class IliasCrawler(HttpCrawler):
# TODO: Proper exception # TODO: Proper exception
raise RuntimeError("Get page failed too often") raise RuntimeError("Get page failed too often")
print(url) print(url)
try:
async with self.session.get(url) as request: async with self.session.get(url) as request:
soup = soupify(await request.read()) soup = soupify(await request.read())
if self._is_logged_in(soup): if self._is_logged_in(soup):
return soup return soup
await self._shibboleth_login.login(self.session) await self._shibboleth_login.login(self.session)
except Exception:
return await self._get_page(url, retries_left - 1)
return await self._get_page(url, retries_left - 1) return await self._get_page(url, retries_left - 1)