mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Retry on more I/O Errors

Browse Source
This commit is contained in:
I-Al-Istannen
2021-04-13 11:32:55 +02:00
parent 14cdfb6a69
commit 1f2af3a290
3 changed files with 31 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
PFERD/ilias/crawler.py
View File

@ -15,7 +15,7 @@ from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit,
import bs4
import requests
from ..errors import FatalException
from ..errors import FatalException, retry_on_io_exception
from ..logging import PrettyLogger
from ..utils import soupify
from .authenticators import IliasAuthenticator
@ -625,6 +625,7 @@ class IliasCrawler:
return results
@retry_on_io_exception(3, "fetching webpage")
def _get_page(self, url: str, params: Dict[str, Any],
retry_count: int = 0) -> bs4.BeautifulSoup:
"""

24
PFERD/ilias/downloader.py
View File

@ -10,6 +10,7 @@ from typing import Callable, List, Optional, Union
import bs4
import requests
from ..errors import retry_on_io_exception
from ..logging import PrettyLogger
from ..organizer import Organizer
from ..tmp_dir import TmpDir
@ -116,26 +117,23 @@ class IliasDownloader:
"""
LOGGER.debug("Downloading %r", info)
if not self._strategy(self._organizer, info):
self._organizer.mark(info.path)
return
tmp_file = self._tmp_dir.new_path()
download_successful = False
for _ in range(0, 3):
try:
if not self._try_download(info, tmp_file):
LOGGER.info("Re-Authenticating due to download failure: %r", info)
self._authenticator.authenticate(self._session)
else:
download_successful = True
break
except IOError as e:
PRETTY.warning(f"I/O Error when downloading ({e}). Retrying...",)
LOGGER.info("Retrying download for %s", info.path)
@retry_on_io_exception(3, "downloading file")
def download_impl() -> bool:
if not self._try_download(info, tmp_file):
LOGGER.info("Re-Authenticating due to download failure: %r", info)
self._authenticator.authenticate(self._session)
raise IOError("Scheduled retry")
else:
return True
if not download_successful:
if not download_impl():
PRETTY.error(f"Download of file {info.path} failed too often! Skipping it...")
return