pferd/PFERD/ilias/downloader.py

82 lines
2.4 KiB
Python
Raw Normal View History

2020-04-20 18:36:40 +02:00
"""Contains a downloader for ILIAS."""
2020-04-20 20:06:21 +02:00
from dataclasses import dataclass, field
2020-04-20 18:36:40 +02:00
from pathlib import Path
2020-04-20 20:06:21 +02:00
from typing import Any, Dict, List
2020-04-20 18:36:40 +02:00
import bs4
import requests
2020-04-20 18:50:23 +02:00
from ..organizer import Organizer
2020-04-20 18:36:40 +02:00
from ..tmp_dir import TmpDir
from ..utils import soupify, stream_to_path
from .authenticators import IliasAuthenticator
class ContentTypeException(Exception):
"""Thrown when the content type of the ilias element can not be handled."""
2020-04-20 20:06:21 +02:00
@dataclass
class IliasDownloadInfo:
"""
This class describes a single file to be downloaded.
"""
path: Path
url: str
parameters: Dict[str, Any] = field(default_factory=dict)
class IliasDownloader:
2020-04-20 18:36:40 +02:00
"""A downloader for ILIAS."""
def __init__(self, tmp_dir: TmpDir, organizer: Organizer, authenticator: IliasAuthenticator):
"""Create a new IliasDownloader."""
self._authenticator = authenticator
self._session = requests.Session()
self._tmp_dir = tmp_dir
self._organizer = organizer
2020-04-20 20:06:21 +02:00
def download_all(self, infos: List[IliasDownloadInfo]) -> None:
"""
Download multiple files one after the other.
"""
2020-04-20 18:36:40 +02:00
2020-04-20 20:06:21 +02:00
for info in infos:
self.download(info)
def download(self, info: IliasDownloadInfo) -> None:
"""
Download a file from ILIAS.
Retries authentication until eternity if it could not fetch the file.
2020-04-20 18:36:40 +02:00
"""
2020-04-20 20:06:21 +02:00
2020-04-20 18:36:40 +02:00
tmp_file = self._tmp_dir.new_file()
2020-04-20 20:06:21 +02:00
while not self._try_download(info, tmp_file):
2020-04-20 18:36:40 +02:00
self._authenticator.authenticate(self._session)
2020-04-20 20:06:21 +02:00
self._organizer.accept_file(tmp_file, info.path)
2020-04-20 18:36:40 +02:00
2020-04-20 20:06:21 +02:00
def _try_download(self, info: IliasDownloadInfo, target: Path) -> bool:
with self._session.get(info.url, params=info.parameters, stream=True) as response:
2020-04-20 19:27:26 +02:00
content_type = response.headers["content-type"]
2020-04-20 18:36:40 +02:00
if content_type.startswith("text/html"):
# Dangit, we're probably not logged in.
2020-04-20 20:06:21 +02:00
if self._is_logged_in(soupify(response)):
2020-04-20 19:27:26 +02:00
raise ContentTypeException("Attempting to download a web page, not a file")
2020-04-20 18:36:40 +02:00
return False
2020-04-20 19:27:26 +02:00
# Yay, we got the file :)
2020-04-20 20:06:21 +02:00
stream_to_path(response, target)
2020-04-20 19:27:26 +02:00
return True
@staticmethod
def _is_logged_in(soup: bs4.BeautifulSoup) -> bool:
2020-04-20 18:36:40 +02:00
userlog = soup.find("li", {"id": "userlog"})
return userlog is not None