Bump version to v2.6.0

Otherwise the name of the course might not be a invalid file name.

.github/workflows/package.yml

@@ -23,7 +23,7 @@ jobs:
         python-version: '3.x'
 
     - name: "Install dependencies"
-      run: "pip install setuptools pyinstaller rich requests beautifulsoup4 -f --upgrade"
+      run: "pip install setuptools keyring pyinstaller rich requests beautifulsoup4 -f --upgrade"
 
     - name: "Install sync_url.py"
       run: "pyinstaller sync_url.py -F"

.gitignore

@@ -8,6 +8,7 @@ build/
 .env
 .vscode
 ilias_cookies.txt
+PFERD.egg-info/
 
 # PyInstaller
 sync_url.spec

PFERD/errors.py

@@ -37,3 +37,21 @@ def swallow_and_print_errors(function: TFun) -> TFun:
             Console().print_exception()
             return None
     return cast(TFun, inner)
+
+
+def retry_on_io_exception(max_retries: int, message: str) -> Callable[[TFun], TFun]:
+    """
+    Decorates a function and retries it on any exception until the max retries count is hit.
+    """
+    def retry(function: TFun) -> TFun:
+        def inner(*args: Any, **kwargs: Any) -> Any:
+            for i in range(0, max_retries):
+                # pylint: disable=broad-except
+                try:
+                    return function(*args, **kwargs)
+                except IOError as error:
+                    PRETTY.warning(f"Error duing operation '{message}': {error}")
+                    PRETTY.warning(
+                        f"Retrying operation '{message}'. Remaining retries: {max_retries - 1 - i}")
+        return cast(TFun, inner)
+    return retry

PFERD/ilias/crawler.py

@@ -15,7 +15,7 @@ from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit,
 import bs4
 import requests
 
-from ..errors import FatalException
+from ..errors import FatalException, retry_on_io_exception
 from ..logging import PrettyLogger
 from ..utils import soupify
 from .authenticators import IliasAuthenticator
@@ -281,7 +281,10 @@ class IliasCrawler:
                     result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
                     continue
 
-                rest_of_name = meeting_name.removeprefix(date_portion_str)
+                rest_of_name = meeting_name
+                if rest_of_name.startswith(date_portion_str):
+                    rest_of_name = rest_of_name[len(date_portion_str):]
+
                 new_name = datetime.datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") \
                     + rest_of_name
                 new_path = Path(folder_path, _sanitize_path_name(new_name))
@@ -622,6 +625,7 @@ class IliasCrawler:
 
         return results
 
+    @retry_on_io_exception(3, "fetching webpage")
     def _get_page(self, url: str, params: Dict[str, Any],
                   retry_count: int = 0) -> bs4.BeautifulSoup:
         """

PFERD/ilias/date_demangler.py

@@ -20,7 +20,7 @@ def demangle_date(date: str) -> Optional[datetime.datetime]:
     "Gestern, HH:MM"
     "Heute, HH:MM"
     "Morgen, HH:MM"
-    "dd. mon.yyyy, HH:MM
+    "dd. mon yyyy, HH:MM
     """
     saved = locale.setlocale(locale.LC_ALL)
     try:

PFERD/ilias/downloader.py

@@ -10,6 +10,7 @@ from typing import Callable, List, Optional, Union
 import bs4
 import requests
 
+from ..errors import retry_on_io_exception
 from ..logging import PrettyLogger
 from ..organizer import Organizer
 from ..tmp_dir import TmpDir
@@ -116,15 +117,25 @@ class IliasDownloader:
         """
 
         LOGGER.debug("Downloading %r", info)
+
         if not self._strategy(self._organizer, info):
             self._organizer.mark(info.path)
             return
 
         tmp_file = self._tmp_dir.new_path()
 
-        while not self._try_download(info, tmp_file):
+        @retry_on_io_exception(3, "downloading file")
-            LOGGER.info("Retrying download: %r", info)
+        def download_impl() -> bool:
+            if not self._try_download(info, tmp_file):
+                LOGGER.info("Re-Authenticating due to download failure: %r", info)
                 self._authenticator.authenticate(self._session)
+                raise IOError("Scheduled retry")
+            else:
+                return True
+
+        if not download_impl():
+            PRETTY.error(f"Download of file {info.path} failed too often! Skipping it...")
+            return
 
         dst_path = self._organizer.accept_file(tmp_file, info.path)
         if dst_path and info.modification_date:

PFERD/ipd.py

@@ -82,7 +82,10 @@ class IpdCrawler:
 
         items: List[IpdDownloadInfo] = []
 
-        for link in page.findAll(name="a", attrs={"href": lambda x: x and x.endswith("pdf")}):
+        def is_relevant_url(x: str) -> bool:
+            return x.endswith(".pdf") or x.endswith(".c") or x.endswith(".java") or x.endswith(".zip")
+
+        for link in page.findAll(name="a", attrs={"href": lambda x: x and is_relevant_url(x)}):
             href: str = link.attrs.get("href")
             name = href.split("/")[-1]
 

README.md

@@ -37,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
 To install PFERD or update your installation to the latest version, run this
 wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
+$ pip install git+https://github.com/Garmelon/PFERD@v2.6.0
 ```
 
 The use of [venv] is recommended.
@@ -59,9 +59,9 @@ A full example setup and initial use could look like:
 $ mkdir Vorlesungen
 $ cd Vorlesungen
 $ python3 -m venv .venv
-$ .venv/bin/activate
+$ source .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
+$ pip install git+https://github.com/Garmelon/PFERD@v2.6.0
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.1/example_config.py
+$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.6.0/example_config.py
 $ python3 example_config.py
 $ deactivate
 ```
@@ -69,7 +69,7 @@ $ deactivate
 Subsequent runs of the program might look like:
 ```
 $ cd Vorlesungen
-$ .venv/bin/activate
+$ source .venv/bin/activate
 $ python3 example_config.py
 $ deactivate
 ```

setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="PFERD",
-    version="2.5.1",
+    version="2.6.0",
     packages=find_packages(),
     install_requires=[
         "requests>=2.21.0",

sync_url.py

@@ -74,7 +74,7 @@ def main() -> None:
                         "one line in the following format: '<user>:<password>'")
     parser.add_argument("-k", "--keyring", action="store_true",
                         help="Use the system keyring service for authentication")
-    parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos")
+    parser.add_argument('--no-videos', action="store_true", help="Don't download videos")
     parser.add_argument('--local-first', action="store_true",
                         help="Don't prompt for confirmation, keep existing files")
     parser.add_argument('--remote-first', action="store_true",
@@ -113,7 +113,7 @@ def main() -> None:
         if not element_name:
             print("Error, could not get element name. Please specify a folder yourself.")
             return
-        folder = Path(element_name)
+        folder = sanitize_windows_path(Path(element_name.replace("/", "-").replace("\\", "-")))
         cookie_jar.save_cookies()
     else:
         folder = Path(args.folder)