Bump version

Otherwise the name of the course might not be a invalid file name.

.gitignore

@@ -8,6 +8,7 @@ build/
 .env
 .vscode
 ilias_cookies.txt
+PFERD.egg-info/
 
 # PyInstaller
 sync_url.spec

PFERD/authenticators.py

@@ -14,7 +14,7 @@ PRETTY = PrettyLogger(LOGGER)
 try:
     import keyring
 except ImportError:
-    PRETTY.warning("Keyring module not found, KeyringAuthenticator won't work!")
+    pass
 
 
 class TfaAuthenticator:

PFERD/ilias/authenticators.py

@@ -74,6 +74,8 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
             form = soup.find("form", {"class": "full content", "method": "post"})
             action = form["action"]
 
+            csrf_token = form.find("input", {"name": "csrf_token"})["value"]
+
             # Equivalent: Enter credentials in
             # https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
             LOGGER.debug("Attempt to log in to Shibboleth using credentials")
@@ -82,6 +84,7 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
                 "_eventId_proceed": "",
                 "j_username": self._auth.username,
                 "j_password": self._auth.password,
+                "csrf_token": csrf_token
             }
             soup = soupify(sess.post(url, data=data))
 

PFERD/ilias/crawler.py

@@ -40,6 +40,7 @@ class IliasElementType(Enum):
     REGULAR_FILE = "REGULAR_FILE"
     VIDEO_FILE = "VIDEO_FILE"
     FORUM = "FORUM"
+    MEETING = "MEETING"
     EXTERNAL_LINK = "EXTERNAL_LINK"
 
     def is_folder(self) -> bool:
@@ -241,6 +242,8 @@ class IliasCrawler:
                 entries_to_process += self._crawl_video_directory(entry.path, url)
                 continue
 
+            PRETTY.warning(f"Unknown type: {entry.entry_type}!")
+
         return result
 
     def _crawl_folder(self, folder_path: Path, url: str) -> List[IliasCrawlerEntry]:
@@ -269,6 +272,25 @@ class IliasCrawler:
 
             if element_type == IliasElementType.REGULAR_FILE:
                 result += self._crawl_file(folder_path, link, abs_url)
+            elif element_type == IliasElementType.MEETING:
+                meeting_name = str(element_path.name)
+                date_portion_str = meeting_name.split(" - ")[0]
+                date_portion = demangle_date(date_portion_str)
+
+                if not date_portion:
+                    result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
+                    continue
+
+                rest_of_name = meeting_name
+                if rest_of_name.startswith(date_portion_str):
+                    rest_of_name = rest_of_name[len(date_portion_str):]
+
+                new_name = datetime.datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") \
+                    + rest_of_name
+                new_path = Path(folder_path, _sanitize_path_name(new_name))
+                result += [
+                    IliasCrawlerEntry(new_path, abs_url, IliasElementType.REGULAR_FOLDER, None)
+                ]
             elif element_type is not None:
                 result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
             else:
@@ -320,6 +342,8 @@ class IliasCrawler:
         """
         # pylint: disable=too-many-return-statements
 
+        found_parent: Optional[bs4.Tag] = None
+
         # We look for the outer div of our inner link, to find information around it
         # (mostly the icon)
         for parent in link_element.parents:
@@ -350,6 +374,9 @@ class IliasCrawler:
         if str(img_tag["src"]).endswith("frm.svg"):
             return IliasElementType.FORUM
 
+        if str(img_tag["src"]).endswith("sess.svg"):
+            return IliasElementType.MEETING
+
         return IliasElementType.REGULAR_FOLDER
 
     @staticmethod

PFERD/ilias/downloader.py

@@ -122,9 +122,22 @@ class IliasDownloader:
 
         tmp_file = self._tmp_dir.new_path()
 
-        while not self._try_download(info, tmp_file):
+        download_successful = False
-            LOGGER.info("Retrying download: %r", info)
+        for _ in range(0, 3):
-            self._authenticator.authenticate(self._session)
+            try:
+                if not self._try_download(info, tmp_file):
+                    LOGGER.info("Re-Authenticating due to download failure: %r", info)
+                    self._authenticator.authenticate(self._session)
+                else:
+                    download_successful = True
+                    break
+            except IOError as e:
+                PRETTY.warning(f"I/O Error when downloading ({e}). Retrying...",)
+            LOGGER.info("Retrying download for %s", info.path)
+
+        if not download_successful:
+            PRETTY.error(f"Download of file {info.path} failed too often! Skipping it...")
+            return
 
         dst_path = self._organizer.accept_file(tmp_file, info.path)
         if dst_path and info.modification_date:

PFERD/logging.py

@@ -3,13 +3,10 @@ Contains a few logger utility functions and implementations.
 """
 
 import logging
-from pathlib import Path
+from typing import Optional
-from typing import List, Optional
 
-from rich import print as rich_print
 from rich._log_render import LogRender
 from rich.console import Console
-from rich.panel import Panel
 from rich.style import Style
 from rich.text import Text
 from rich.theme import Theme

PFERD/organizer.py

@@ -116,7 +116,7 @@ class Organizer(Location):
         if self._is_marked(dst):
             PRETTY.warning(f"File {str(dst_absolute)!r} was already written!")
             conflict = ConflictType.MARKED_FILE_OVERWRITTEN
-            if self._resolve_conflict(f"Overwrite file?", dst_absolute, conflict, default=False):
+            if self._resolve_conflict("Overwrite file?", dst_absolute, conflict, default=False):
                 PRETTY.ignored_file(dst_absolute, "file was written previously")
                 return None
 

README.md

@@ -37,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
 To install PFERD or update your installation to the latest version, run this
 wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
+$ pip install git+https://github.com/Garmelon/PFERD@v2.5.4
 ```
 
 The use of [venv] is recommended.
@@ -59,9 +59,9 @@ A full example setup and initial use could look like:
 $ mkdir Vorlesungen
 $ cd Vorlesungen
 $ python3 -m venv .venv
-$ .venv/bin/activate
+$ source .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
+$ pip install git+https://github.com/Garmelon/PFERD@v2.5.4
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.0/example_config.py
+$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.4/example_config.py
 $ python3 example_config.py
 $ deactivate
 ```
@@ -69,7 +69,7 @@ $ deactivate
 Subsequent runs of the program might look like:
 ```
 $ cd Vorlesungen
-$ .venv/bin/activate
+$ source .venv/bin/activate
 $ python3 example_config.py
 $ deactivate
 ```

setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="PFERD",
-    version="2.5.0",
+    version="2.5.4",
     packages=find_packages(),
     install_requires=[
         "requests>=2.21.0",

sync_url.py

@@ -113,7 +113,7 @@ def main() -> None:
         if not element_name:
             print("Error, could not get element name. Please specify a folder yourself.")
             return
-        folder = Path(element_name)
+        folder = sanitize_windows_path(Path(element_name.replace("/", "-").replace("\\", "-")))
         cookie_jar.save_cookies()
     else:
         folder = Path(args.folder)