mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Prevent too many retries when fetching an ILIAS page

Browse Source
This commit is contained in:
I-Al-Istannen 2020-11-04 22:20:40 +01:00
parent 6f30adcd22
commit 316b9d7bf4
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
PFERD/ilias/crawler.py
View File

@ -593,10 +593,17 @@ class IliasCrawler:
return results return results
def _get_page(self, url: str, params: Dict[str, Any]) -> bs4.BeautifulSoup: def _get_page(self, url: str, params: Dict[str, Any],
retry_count: int = 0) -> bs4.BeautifulSoup:
""" """
Fetches a page from ILIAS, authenticating when needed. Fetches a page from ILIAS, authenticating when needed.
""" """
if retry_count >= 4:
raise FatalException("Could not get a proper page after 4 tries. "
"Maybe your URL is wrong, authentication fails continuously, "
"your ILIAS connection is spotty or ILIAS is not well.")
LOGGER.debug("Fetching %r", url) LOGGER.debug("Fetching %r", url)
response = self._session.get(url, params=params) response = self._session.get(url, params=params)
@ -617,7 +624,7 @@ class IliasCrawler:
self._authenticator.authenticate(self._session) self._authenticator.authenticate(self._session)
return self._get_page(url, params) return self._get_page(url, params, retry_count + 1)
@staticmethod @staticmethod
def _is_logged_in(soup: bs4.BeautifulSoup) -> bool: def _is_logged_in(soup: bs4.BeautifulSoup) -> bool: