Bump version to v2.6.2

Otherwise the name of the course might not be a invalid file name.

.github/workflows/package.yml

@@ -23,7 +23,7 @@ jobs:
         python-version: '3.x'
 
     - name: "Install dependencies"
-      run: "pip install setuptools pyinstaller rich requests beautifulsoup4 -f --upgrade"
+      run: "pip install setuptools keyring pyinstaller rich requests beautifulsoup4 -f --upgrade"
 
     - name: "Install sync_url.py"
       run: "pyinstaller sync_url.py -F"

.gitignore

@@ -8,6 +8,7 @@ build/
 .env
 .vscode
 ilias_cookies.txt
+PFERD.egg-info/
 
 # PyInstaller
 sync_url.spec

PFERD/errors.py

@@ -37,3 +37,21 @@ def swallow_and_print_errors(function: TFun) -> TFun:
             Console().print_exception()
             return None
     return cast(TFun, inner)
+
+
+def retry_on_io_exception(max_retries: int, message: str) -> Callable[[TFun], TFun]:
+    """
+    Decorates a function and retries it on any exception until the max retries count is hit.
+    """
+    def retry(function: TFun) -> TFun:
+        def inner(*args: Any, **kwargs: Any) -> Any:
+            for i in range(0, max_retries):
+                # pylint: disable=broad-except
+                try:
+                    return function(*args, **kwargs)
+                except IOError as error:
+                    PRETTY.warning(f"Error duing operation '{message}': {error}")
+                    PRETTY.warning(
+                        f"Retrying operation '{message}'. Remaining retries: {max_retries - 1 - i}")
+        return cast(TFun, inner)
+    return retry

PFERD/ilias/crawler.py

@@ -15,7 +15,7 @@ from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit,
 import bs4
 import requests
 
-from ..errors import FatalException
+from ..errors import FatalException, retry_on_io_exception
 from ..logging import PrettyLogger
 from ..utils import soupify
 from .authenticators import IliasAuthenticator
@@ -281,7 +281,10 @@ class IliasCrawler:
                     result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
                     continue
 
-                rest_of_name = meeting_name.removeprefix(date_portion_str)
+                rest_of_name = meeting_name
+                if rest_of_name.startswith(date_portion_str):
+                    rest_of_name = rest_of_name[len(date_portion_str):]
+
                 new_name = datetime.datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") \
                     + rest_of_name
                 new_path = Path(folder_path, _sanitize_path_name(new_name))
@@ -622,6 +625,7 @@ class IliasCrawler:
 
         return results
 
+    @retry_on_io_exception(3, "fetching webpage")
     def _get_page(self, url: str, params: Dict[str, Any],
                   retry_count: int = 0) -> bs4.BeautifulSoup:
         """

PFERD/ilias/date_demangler.py

@@ -20,7 +20,7 @@ def demangle_date(date: str) -> Optional[datetime.datetime]:
     "Gestern, HH:MM"
     "Heute, HH:MM"
     "Morgen, HH:MM"
-    "dd. mon.yyyy, HH:MM
+    "dd. mon yyyy, HH:MM
     """
     saved = locale.setlocale(locale.LC_ALL)
     try:

PFERD/ilias/downloader.py

@@ -10,6 +10,7 @@ from typing import Callable, List, Optional, Union
 import bs4
 import requests
 
+from ..errors import retry_on_io_exception
 from ..logging import PrettyLogger
 from ..organizer import Organizer
 from ..tmp_dir import TmpDir
@@ -116,15 +117,25 @@ class IliasDownloader:
         """
 
         LOGGER.debug("Downloading %r", info)
+
         if not self._strategy(self._organizer, info):
             self._organizer.mark(info.path)
             return
 
         tmp_file = self._tmp_dir.new_path()
 
-        while not self._try_download(info, tmp_file):
-            LOGGER.info("Retrying download: %r", info)
+        @retry_on_io_exception(3, "downloading file")
+        def download_impl() -> bool:
+            if not self._try_download(info, tmp_file):
+                LOGGER.info("Re-Authenticating due to download failure: %r", info)
                 self._authenticator.authenticate(self._session)
+                raise IOError("Scheduled retry")
+            else:
+                return True
+
+        if not download_impl():
+            PRETTY.error(f"Download of file {info.path} failed too often! Skipping it...")
+            return
 
         dst_path = self._organizer.accept_file(tmp_file, info.path)
         if dst_path and info.modification_date:

PFERD/ipd.py

@@ -82,7 +82,10 @@ class IpdCrawler:
 
         items: List[IpdDownloadInfo] = []
 
-        for link in page.findAll(name="a", attrs={"href": lambda x: x and x.endswith("pdf")}):
+        def is_relevant_url(x: str) -> bool:
+            return x.endswith(".pdf") or x.endswith(".c") or x.endswith(".java") or x.endswith(".zip")
+
+        for link in page.findAll(name="a", attrs={"href": lambda x: x and is_relevant_url(x)}):
             href: str = link.attrs.get("href")
             name = href.split("/")[-1]
 

PFERD/transform.py

@@ -130,7 +130,7 @@ def re_rename(regex: Regex, target: str) -> Transform:
     return inner
 
 
-def sanitize_windows_path(path: PurePath) -> Optional[PurePath]:
+def sanitize_windows_path(path: PurePath) -> PurePath:
     """
     A small function to escape characters that are forbidden in windows path names.
     This method is a no-op on other operating systems.

README.md

@@ -37,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
 To install PFERD or update your installation to the latest version, run this
 wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
+$ pip install git+https://github.com/Garmelon/PFERD@v2.6.2
 ```
 
 The use of [venv] is recommended.
@@ -59,9 +59,9 @@ A full example setup and initial use could look like:
 $ mkdir Vorlesungen
 $ cd Vorlesungen
 $ python3 -m venv .venv
-$ .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.1/example_config.py
+$ source .venv/bin/activate
+$ pip install git+https://github.com/Garmelon/PFERD@v2.6.2
+$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.6.2/example_config.py
 $ python3 example_config.py
 $ deactivate
 ```
@@ -69,7 +69,7 @@ $ deactivate
 Subsequent runs of the program might look like:
 ```
 $ cd Vorlesungen
-$ .venv/bin/activate
+$ source .venv/bin/activate
 $ python3 example_config.py
 $ deactivate
 ```

setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="PFERD",
-    version="2.5.1",
+    version="2.6.2",
     packages=find_packages(),
     install_requires=[
         "requests>=2.21.0",

sync_url.py

@@ -26,9 +26,10 @@ _LOGGER = logging.getLogger("sync_url")
 _PRETTY = PrettyLogger(_LOGGER)
 
 
-def _extract_credentials(file_path: Optional[str]) -> UserPassAuthenticator:
+def _extract_credentials(file_path: Optional[str],
+                         username: Optional[str], password: Optional[str]) -> UserPassAuthenticator:
     if not file_path:
-        return UserPassAuthenticator("KIT ILIAS Shibboleth", None, None)
+        return UserPassAuthenticator("KIT ILIAS Shibboleth", username, password)
 
     if not Path(file_path).exists():
         _PRETTY.error("Credential file does not exist")
@@ -74,7 +75,7 @@ def main() -> None:
                         "one line in the following format: '<user>:<password>'")
     parser.add_argument("-k", "--keyring", action="store_true",
                         help="Use the system keyring service for authentication")
-    parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos")
+    parser.add_argument('--no-videos', action="store_true", help="Don't download videos")
     parser.add_argument('--local-first', action="store_true",
                         help="Don't prompt for confirmation, keep existing files")
     parser.add_argument('--remote-first', action="store_true",
@@ -96,7 +97,7 @@ def main() -> None:
             "KIT ILIAS Shibboleth", username=args.username, password=args.password
         )
     else:
-        inner_auth = _extract_credentials(args.credential_file)
+        inner_auth = _extract_credentials(args.credential_file, args.username, args.password)
 
     username, password = inner_auth.get_credentials()
     authenticator = KitShibbolethAuthenticator(inner_auth)
@@ -113,7 +114,7 @@ def main() -> None:
         if not element_name:
             print("Error, could not get element name. Please specify a folder yourself.")
             return
-        folder = Path(element_name)
+        folder = sanitize_windows_path(Path(element_name.replace("/", "-").replace("\\", "-")))
         cookie_jar.save_cookies()
     else:
         folder = Path(args.folder)
@@ -121,7 +122,8 @@ def main() -> None:
     # files may not escape the pferd_root with relative paths
     # note: Path(Path.cwd, Path(folder)) == Path(folder) if it is an absolute path
     pferd_root = Path(Path.cwd(), Path(folder)).parent
-    target = folder.name
+    # Folder might be a *PurePath* at this point
+    target = Path(folder).resolve().name
     pferd = Pferd(pferd_root, test_run=args.test_run)
 
     def dir_filter(_: Path, element: IliasElementType) -> bool:
@@ -130,13 +132,13 @@ def main() -> None:
         return True
 
     if args.local_first:
-        file_confilict_resolver: FileConflictResolver = _resolve_local_first
+        file_conflict_resolver: FileConflictResolver = _resolve_local_first
     elif args.no_delete:
-        file_confilict_resolver = _resolve_no_delete
+        file_conflict_resolver = _resolve_no_delete
     elif args.remote_first:
-        file_confilict_resolver = _resolve_remote_first
+        file_conflict_resolver = _resolve_remote_first
     else:
-        file_confilict_resolver = resolve_prompt_user
+        file_conflict_resolver = resolve_prompt_user
 
     pferd.enable_logging()
 
@@ -148,7 +150,7 @@ def main() -> None:
         dir_filter=dir_filter,
         username=username,
         password=password,
-        file_conflict_resolver=file_confilict_resolver,
+        file_conflict_resolver=file_conflict_resolver,
         transform=sanitize_windows_path
     )