diff --git a/PFERD/ilias/downloader.py b/PFERD/ilias/downloader.py index f26c43c..82527a0 100644 --- a/PFERD/ilias/downloader.py +++ b/PFERD/ilias/downloader.py @@ -84,9 +84,13 @@ class IliasDownloader: session: requests.Session, authenticator: IliasAuthenticator, strategy: IliasDownloadStrategy, + timeout: int = 5 ): """ Create a new IliasDownloader. + + The timeout applies to the download request only, as bwcloud uses IPv6 + and requests has a problem with that: https://github.com/psf/requests/issues/5522 """ self._tmp_dir = tmp_dir @@ -94,6 +98,7 @@ class IliasDownloader: self._session = session self._authenticator = authenticator self._strategy = strategy + self._timeout = timeout def download_all(self, infos: List[IliasDownloadInfo]) -> None: """ @@ -137,7 +142,7 @@ class IliasDownloader: PRETTY.warning(f"Could not download {str(info.path)!r} as I got no URL :/") return True - with self._session.get(url, stream=True) as response: + with self._session.get(url, stream=True, timeout=self._timeout) as response: content_type = response.headers["content-type"] has_content_disposition = "content-disposition" in response.headers diff --git a/PFERD/pferd.py b/PFERD/pferd.py index e99cf20..0b25151 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -72,7 +72,8 @@ class Pferd(Location): dir_filter: IliasDirectoryFilter, transform: Transform, download_strategy: IliasDownloadStrategy, - clean: bool = True + timeout: int, + clean: bool = True, ) -> Organizer: # pylint: disable=too-many-locals cookie_jar = CookieJar(to_path(cookies) if cookies else None) @@ -81,7 +82,8 @@ class Pferd(Location): organizer = Organizer(self.resolve(to_path(target))) crawler = IliasCrawler(base_url, session, authenticator, dir_filter) - downloader = IliasDownloader(tmp_dir, organizer, session, authenticator, download_strategy) + downloader = IliasDownloader(tmp_dir, organizer, session, + authenticator, download_strategy, timeout) cookie_jar.load_cookies() info = crawl_function(crawler) @@ -112,6 +114,7 @@ class Pferd(Location): password: Optional[str] = None, download_strategy: IliasDownloadStrategy = download_modified_or_new, clean: bool = True, + timeout: int = 5, ) -> Organizer: """ Synchronizes a folder with the ILIAS instance of the KIT. @@ -137,6 +140,8 @@ class Pferd(Location): be downloaded. Can save bandwidth and reduce the number of requests. (default: {download_modified_or_new}) clean {bool} -- Whether to clean up when the method finishes. + timeout {int} -- The download timeout for opencast videos. Sadly needed due to a + requests bug. """ # This authenticator only works with the KIT ilias instance. authenticator = KitShibbolethAuthenticator(username=username, password=password) @@ -152,6 +157,7 @@ class Pferd(Location): transform=transform, download_strategy=download_strategy, clean=clean, + timeout=timeout ) self._download_summary.merge(organizer.download_summary) @@ -175,6 +181,7 @@ class Pferd(Location): password: Optional[str] = None, download_strategy: IliasDownloadStrategy = download_modified_or_new, clean: bool = True, + timeout: int = 5, ) -> Organizer: """ Synchronizes a folder with the ILIAS instance of the KIT. This method will crawl the ILIAS @@ -199,6 +206,8 @@ class Pferd(Location): be downloaded. Can save bandwidth and reduce the number of requests. (default: {download_modified_or_new}) clean {bool} -- Whether to clean up when the method finishes. + timeout {int} -- The download timeout for opencast videos. Sadly needed due to a + requests bug. """ # This authenticator only works with the KIT ilias instance. authenticator = KitShibbolethAuthenticator(username=username, password=password) @@ -214,6 +223,7 @@ class Pferd(Location): transform=transform, download_strategy=download_strategy, clean=clean, + timeout=timeout ) self._download_summary.merge(organizer.download_summary)