From cf6903d109fead73a622351f98d24b05d013e93a Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Sat, 15 May 2021 22:46:26 +0200 Subject: [PATCH] Retry crawling on I/O failure --- PFERD/crawlers/ilias.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/PFERD/crawlers/ilias.py b/PFERD/crawlers/ilias.py index e52d329..b3190c6 100644 --- a/PFERD/crawlers/ilias.py +++ b/PFERD/crawlers/ilias.py @@ -596,12 +596,15 @@ class IliasCrawler(HttpCrawler): # TODO: Proper exception raise RuntimeError("Get page failed too often") print(url) - async with self.session.get(url) as request: - soup = soupify(await request.read()) - if self._is_logged_in(soup): - return soup + try: + async with self.session.get(url) as request: + soup = soupify(await request.read()) + if self._is_logged_in(soup): + return soup - await self._shibboleth_login.login(self.session) + await self._shibboleth_login.login(self.session) + except Exception: + return await self._get_page(url, retries_left - 1) return await self._get_page(url, retries_left - 1)