Bump version

Add flag to make sync_url use defaults instead of prompting

.github/workflows/package.yml

@@ -67,7 +67,7 @@ jobs:
     - name: "Upload release artifacts"
       uses: softprops/action-gh-release@v1
       with:
-        body: "Download the correct sync_url for your platform and run it in the terminal or CMD. You might need to make it executable on Linux/Mac with `chmod +x <file>`."
+        body: "Download the correct sync_url for your platform and run it in the terminal or CMD. You might need to make it executable on Linux/Mac with `chmod +x <file>`. Also please enclose the *url you pass to the program in double quotes* or your shell might silently screw it up!"
         files: |
           pferd_sync_url_mac
           pferd_sync_url_linux

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw
+Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw, TheChristophe, Scriptim
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

PFERD/authenticators.py

@@ -3,8 +3,19 @@ General authenticators useful in many situations
 """
 
 import getpass
+import logging
 from typing import Optional, Tuple
 
+from .logging import PrettyLogger
+
+LOGGER = logging.getLogger(__name__)
+PRETTY = PrettyLogger(LOGGER)
+
+try:
+    import keyring
+except ImportError:
+    PRETTY.warning("Keyring module not found, KeyringAuthenticator won't work!")
+
 
 class TfaAuthenticator:
     # pylint: disable=too-few-public-methods
@@ -123,3 +134,81 @@ class UserPassAuthenticator:
         if self._given_username is not None and self._given_password is not None:
             self._given_username = None
             self._given_password = None
+
+
+class KeyringAuthenticator(UserPassAuthenticator):
+    """
+    An authenticator for username-password combinations that stores the
+    password using the system keyring service and prompts the user for missing
+    information.
+    """
+
+    def get_credentials(self) -> Tuple[str, str]:
+        """
+        Returns a tuple (username, password). Prompts user for username or
+        password when necessary.
+        """
+
+        if self._username is None and self._given_username is not None:
+            self._username = self._given_username
+
+        if self._password is None and self._given_password is not None:
+            self._password = self._given_password
+
+        if self._username is not None and self._password is None:
+            self._load_password()
+
+        if self._username is None or self._password is None:
+            print(f"Enter credentials ({self._reason})")
+
+        username: str
+        if self._username is None:
+            username = input("Username: ")
+            self._username = username
+        else:
+            username = self._username
+
+        if self._password is None:
+            self._load_password()
+
+        password: str
+        if self._password is None:
+            password = getpass.getpass(prompt="Password: ")
+            self._password = password
+            self._save_password()
+        else:
+            password = self._password
+
+        return (username, password)
+
+    def _load_password(self) -> None:
+        """
+        Loads the saved password associated with self._username from the system
+        keyring service (or None if not password has been saved yet) and stores
+        it in self._password.
+        """
+        self._password = keyring.get_password("pferd-ilias", self._username)
+
+    def _save_password(self) -> None:
+        """
+        Saves self._password to the system keyring service and associates it
+        with self._username.
+        """
+        keyring.set_password("pferd-ilias", self._username, self._password)
+
+    def invalidate_credentials(self) -> None:
+        """
+        Marks the credentials as invalid. If only a username was supplied in
+        the constructor, assumes that the username is valid and only the
+        password is invalid. If only a password was supplied in the
+        constructor, assumes that the password is valid and only the username
+        is invalid. Otherwise, assumes that username and password are both
+        invalid.
+        """
+
+        try:
+            keyring.delete_password("pferd-ilias", self._username)
+        except keyring.errors.PasswordDeleteError:
+            pass
+
+        super().invalidate_credentials()

PFERD/download_summary.py

@@ -5,6 +5,12 @@ from pathlib import Path
 from typing import List
 
 
+def _mergeNoDuplicate(first: List[Path], second: List[Path]) -> List[Path]:
+    tmp = list(set(first + second))
+    tmp.sort(key=lambda x: str(x.resolve()))
+    return tmp
+
+
 class DownloadSummary:
     """
     Keeps track of all new, modified or deleted files and provides a summary.
@@ -40,9 +46,9 @@ class DownloadSummary:
         """
         Merges ourselves with the passed summary. Modifies this object, but not the passed one.
         """
-        self._new_files += summary.new_files
+        self._new_files = _mergeNoDuplicate(self._new_files, summary.new_files)
-        self._modified_files += summary.modified_files
+        self._modified_files = _mergeNoDuplicate(self._modified_files, summary.modified_files)
-        self._deleted_files += summary.deleted_files
+        self._deleted_files = _mergeNoDuplicate(self._deleted_files, summary.deleted_files)
 
     def add_deleted_file(self, path: Path) -> None:
         """

PFERD/ilias/authenticators.py

@@ -37,8 +37,12 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
     Authenticate via KIT's shibboleth system.
     """
 
-    def __init__(self, username: Optional[str] = None, password: Optional[str] = None) -> None:
+    def __init__(self, authenticator: Optional[UserPassAuthenticator] = None) -> None:
-        self._auth = UserPassAuthenticator("KIT ILIAS Shibboleth", username, password)
+        if authenticator:
+            self._auth = authenticator
+        else:
+            self._auth = UserPassAuthenticator("KIT ILIAS Shibboleth")
+
         self._tfa_auth = TfaAuthenticator("KIT ILIAS Shibboleth")
 
     def authenticate(self, sess: requests.Session) -> None:

PFERD/ilias/crawler.py

@@ -26,6 +26,10 @@ LOGGER = logging.getLogger(__name__)
 PRETTY = PrettyLogger(LOGGER)
 
 
+def _sanitize_path_name(name: str) -> str:
+    return name.replace("/", "-").replace("\\", "-")
+
+
 class IliasElementType(Enum):
     """
     The type of an ilias element.
@@ -260,7 +264,7 @@ class IliasCrawler:
         links: List[bs4.Tag] = soup.select("a.il_ContainerItemTitle")
         for link in links:
             abs_url = self._abs_url_from_link(link)
-            element_path = Path(folder_path, link.getText().strip())
+            element_path = Path(folder_path, _sanitize_path_name(link.getText().strip()))
             element_type = self._find_type_from_link(element_path, link, abs_url)
 
             if element_type == IliasElementType.REGULAR_FILE:
@@ -377,7 +381,7 @@ class IliasCrawler:
             modification_date = demangle_date(modification_date_str)
 
         # Grab the name from the link text
-        name = link_element.getText()
+        name = _sanitize_path_name(link_element.getText())
         full_path = Path(path, name + "." + file_type)
 
         return [
@@ -508,7 +512,7 @@ class IliasCrawler:
         ).getText().strip()
         title += ".mp4"
 
-        video_path: Path = Path(parent_path, title)
+        video_path: Path = Path(parent_path, _sanitize_path_name(title))
 
         video_url = self._abs_url_from_link(link)
 
@@ -580,6 +584,7 @@ class IliasCrawler:
                 # Two divs, side by side. Left is the name, right is the link ==> get left
                 # sibling
                 file_name = file_link.parent.findPrevious(name="div").getText().strip()
+                file_name = _sanitize_path_name(file_name)
                 url = self._abs_url_from_link(file_link)
 
                 LOGGER.debug("Found file %r at %r", file_name, url)
@@ -593,10 +598,17 @@ class IliasCrawler:
 
         return results
 
-    def _get_page(self, url: str, params: Dict[str, Any]) -> bs4.BeautifulSoup:
+    def _get_page(self, url: str, params: Dict[str, Any],
+                  retry_count: int = 0) -> bs4.BeautifulSoup:
         """
         Fetches a page from ILIAS, authenticating when needed.
         """
+
+        if retry_count >= 4:
+            raise FatalException("Could not get a proper page after 4 tries. "
+                                 "Maybe your URL is wrong, authentication fails continuously, "
+                                 "your ILIAS connection is spotty or ILIAS is not well.")
+
         LOGGER.debug("Fetching %r", url)
 
         response = self._session.get(url, params=params)
@@ -617,7 +629,7 @@ class IliasCrawler:
 
         self._authenticator.authenticate(self._session)
 
-        return self._get_page(url, params)
+        return self._get_page(url, params, retry_count + 1)
 
     @staticmethod
     def _is_logged_in(soup: bs4.BeautifulSoup) -> bool:

PFERD/ipd.py

@@ -0,0 +1,151 @@
+"""
+Utility functions and a scraper/downloader for the IPD pages.
+"""
+import datetime
+import logging
+import math
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, List, Optional
+from urllib.parse import urljoin
+
+import bs4
+import requests
+
+from PFERD.errors import FatalException
+from PFERD.utils import soupify
+
+from .logging import PrettyLogger
+from .organizer import Organizer
+from .tmp_dir import TmpDir
+from .transform import Transformable
+from .utils import stream_to_path
+
+LOGGER = logging.getLogger(__name__)
+PRETTY = PrettyLogger(LOGGER)
+
+
+@dataclass
+class IpdDownloadInfo(Transformable):
+    """
+    Information about an ipd entry.
+    """
+    url: str
+    modification_date: Optional[datetime.datetime]
+
+
+IpdDownloadStrategy = Callable[[Organizer, IpdDownloadInfo], bool]
+
+
+def ipd_download_new_or_modified(organizer: Organizer, info: IpdDownloadInfo) -> bool:
+    """
+    Accepts new files or files with a more recent modification date.
+    """
+    resolved_file = organizer.resolve(info.path)
+    if not resolved_file.exists():
+        return True
+    if not info.modification_date:
+        PRETTY.ignored_file(info.path, "could not find modification time, file exists")
+        return False
+
+    resolved_mod_time_seconds = resolved_file.stat().st_mtime
+
+    # Download if the info is newer
+    if info.modification_date.timestamp() > resolved_mod_time_seconds:
+        return True
+
+    PRETTY.ignored_file(info.path, "local file has newer or equal modification time")
+    return False
+
+
+class IpdCrawler:
+    # pylint: disable=too-few-public-methods
+    """
+    A crawler for IPD pages.
+    """
+
+    def __init__(self, base_url: str):
+        self._base_url = base_url
+
+    def _abs_url_from_link(self, link_tag: bs4.Tag) -> str:
+        """
+        Create an absolute url from an <a> tag.
+        """
+        return urljoin(self._base_url, link_tag.get("href"))
+
+    def crawl(self) -> List[IpdDownloadInfo]:
+        """
+        Crawls the playlist given in the constructor.
+        """
+        page = soupify(requests.get(self._base_url))
+
+        items: List[IpdDownloadInfo] = []
+
+        for link in page.findAll(name="a", attrs={"href": lambda x: x and x.endswith("pdf")}):
+            href: str = link.attrs.get("href")
+            name = href.split("/")[-1]
+
+            modification_date: Optional[datetime.datetime] = None
+            try:
+                enclosing_row: bs4.Tag = link.findParent(name="tr")
+                if enclosing_row:
+                    date_text = enclosing_row.find(name="td").text
+                    modification_date = datetime.datetime.strptime(date_text, "%d.%m.%Y")
+            except ValueError:
+                modification_date = None
+
+            items.append(IpdDownloadInfo(
+                Path(name),
+                url=self._abs_url_from_link(link),
+                modification_date=modification_date
+            ))
+
+        return items
+
+
+class IpdDownloader:
+    """
+    A downloader for ipd files.
+    """
+
+    def __init__(self, tmp_dir: TmpDir, organizer: Organizer, strategy: IpdDownloadStrategy):
+        self._tmp_dir = tmp_dir
+        self._organizer = organizer
+        self._strategy = strategy
+        self._session = requests.session()
+
+    def download_all(self, infos: List[IpdDownloadInfo]) -> None:
+        """
+        Download multiple files one after the other.
+        """
+        for info in infos:
+            self.download(info)
+
+    def download(self, info: IpdDownloadInfo) -> None:
+        """
+        Download a single file.
+        """
+        if not self._strategy(self._organizer, info):
+            self._organizer.mark(info.path)
+            return
+
+        with self._session.get(info.url, stream=True) as response:
+            if response.status_code == 200:
+                tmp_file = self._tmp_dir.new_path()
+                stream_to_path(response, tmp_file, info.path.name)
+                dst_path = self._organizer.accept_file(tmp_file, info.path)
+
+                if dst_path and info.modification_date:
+                    os.utime(
+                        dst_path,
+                        times=(
+                            math.ceil(info.modification_date.timestamp()),
+                            math.ceil(info.modification_date.timestamp())
+                        )
+                    )
+
+            elif response.status_code == 403:
+                raise FatalException("Received 403. Are you not using the KIT VPN?")
+            else:
+                PRETTY.warning(f"Could not download file, got response {response.status_code}")

PFERD/organizer.py

@@ -7,8 +7,9 @@ import filecmp
 import logging
 import os
 import shutil
+from enum import Enum
 from pathlib import Path, PurePath
-from typing import List, Optional, Set
+from typing import Callable, List, Optional, Set
 
 from .download_summary import DownloadSummary
 from .location import Location
@@ -19,6 +20,51 @@ LOGGER = logging.getLogger(__name__)
 PRETTY = PrettyLogger(LOGGER)
 
 
+class ConflictType(Enum):
+    """
+    The type of the conflict. A file might not exist anymore and will be deleted
+    or it might be overwritten with a newer version.
+
+    FILE_OVERWRITTEN: An existing file will be updated
+    MARKED_FILE_OVERWRITTEN: A file is written for the second+ time in this run
+    FILE_DELETED: The file was deleted
+    """
+    FILE_OVERWRITTEN = "overwritten"
+    MARKED_FILE_OVERWRITTEN = "marked_file_overwritten"
+    FILE_DELETED = "deleted"
+
+
+class FileConflictResolution(Enum):
+    """
+    The reaction when confronted with a file conflict:
+
+    DESTROY_EXISTING: Delete/overwrite the current file
+    KEEP_EXISTING: Keep the current file
+    DEFAULT: Do whatever the PFERD authors thought is sensible
+    PROMPT: Interactively ask the user
+    """
+
+    DESTROY_EXISTING = "destroy"
+
+    KEEP_EXISTING = "keep"
+
+    DEFAULT = "default"
+
+    PROMPT = "prompt"
+
+
+FileConflictResolver = Callable[[PurePath, ConflictType], FileConflictResolution]
+
+
+def resolve_prompt_user(_path: PurePath, conflict: ConflictType) -> FileConflictResolution:
+    """
+    Resolves conflicts by asking the user if a file was written twice or will be deleted.
+    """
+    if conflict == ConflictType.FILE_OVERWRITTEN:
+        return FileConflictResolution.DESTROY_EXISTING
+    return FileConflictResolution.PROMPT
+
+
 class FileAcceptException(Exception):
     """An exception while accepting a file."""
 
@@ -26,7 +72,7 @@ class FileAcceptException(Exception):
 class Organizer(Location):
     """A helper for managing downloaded files."""
 
-    def __init__(self, path: Path):
+    def __init__(self, path: Path, conflict_resolver: FileConflictResolver = resolve_prompt_user):
         """Create a new organizer for a given path."""
         super().__init__(path)
         self._known_files: Set[Path] = set()
@@ -36,6 +82,8 @@ class Organizer(Location):
 
         self.download_summary = DownloadSummary()
 
+        self.conflict_resolver = conflict_resolver
+
     def accept_file(self, src: Path, dst: PurePath) -> Optional[Path]:
         """
         Move a file to this organizer and mark it.
@@ -67,13 +115,16 @@ class Organizer(Location):
 
         if self._is_marked(dst):
             PRETTY.warning(f"File {str(dst_absolute)!r} was already written!")
-            if not prompt_yes_no(f"Overwrite file?", default=False):
+            conflict = ConflictType.MARKED_FILE_OVERWRITTEN
+            if self._resolve_conflict(f"Overwrite file?", dst_absolute, conflict, default=False):
                 PRETTY.ignored_file(dst_absolute, "file was written previously")
                 return None
 
         # Destination file is directory
         if dst_absolute.exists() and dst_absolute.is_dir():
-            if prompt_yes_no(f"Overwrite folder {dst_absolute} with file?", default=False):
+            prompt = f"Overwrite folder {dst_absolute} with file?"
+            conflict = ConflictType.FILE_OVERWRITTEN
+            if self._resolve_conflict(prompt, dst_absolute, conflict, default=False):
                 shutil.rmtree(dst_absolute)
             else:
                 PRETTY.warning(f"Could not add file {str(dst_absolute)!r}")
@@ -87,6 +138,12 @@ class Organizer(Location):
                 self.mark(dst)
                 return dst_absolute
 
+            prompt = f"Overwrite file {dst_absolute}?"
+            conflict = ConflictType.FILE_OVERWRITTEN
+            if not self._resolve_conflict(prompt, dst_absolute, conflict, default=True):
+                PRETTY.ignored_file(dst_absolute, "user conflict resolution")
+                return None
+
             self.download_summary.add_modified_file(dst_absolute)
             PRETTY.modified_file(dst_absolute)
         else:
@@ -124,6 +181,8 @@ class Organizer(Location):
         self._cleanup(self.path)
 
     def _cleanup(self, start_dir: Path) -> None:
+        if not start_dir.exists():
+            return
         paths: List[Path] = list(start_dir.iterdir())
 
         # Recursively clean paths
@@ -142,6 +201,24 @@ class Organizer(Location):
     def _delete_file_if_confirmed(self, path: Path) -> None:
         prompt = f"Do you want to delete {path}"
 
-        if prompt_yes_no(prompt, False):
+        if self._resolve_conflict(prompt, path, ConflictType.FILE_DELETED, default=False):
             self.download_summary.add_deleted_file(path)
             path.unlink()
+        else:
+            PRETTY.ignored_file(path, "user conflict resolution")
+
+    def _resolve_conflict(
+            self, prompt: str, path: Path, conflict: ConflictType, default: bool
+    ) -> bool:
+        if not self.conflict_resolver:
+            return prompt_yes_no(prompt, default=default)
+
+        result = self.conflict_resolver(path, conflict)
+        if result == FileConflictResolution.DEFAULT:
+            return default
+        if result == FileConflictResolution.KEEP_EXISTING:
+            return False
+        if result == FileConflictResolution.DESTROY_EXISTING:
+            return True
+
+        return prompt_yes_no(prompt, default=default)

PFERD/pferd.py

@@ -6,6 +6,7 @@ import logging
 from pathlib import Path
 from typing import Callable, List, Optional, Union
 
+from .authenticators import UserPassAuthenticator
 from .cookie_jar import CookieJar
 from .diva import (DivaDownloader, DivaDownloadStrategy, DivaPlaylistCrawler,
                    diva_download_new)
@@ -14,9 +15,11 @@ from .errors import FatalException, swallow_and_print_errors
 from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
                     IliasDownloader, IliasDownloadInfo, IliasDownloadStrategy,
                     KitShibbolethAuthenticator, download_modified_or_new)
+from .ipd import (IpdCrawler, IpdDownloader, IpdDownloadInfo,
+                  IpdDownloadStrategy, ipd_download_new_or_modified)
 from .location import Location
 from .logging import PrettyLogger, enable_logging
-from .organizer import Organizer
+from .organizer import FileConflictResolver, Organizer, resolve_prompt_user
 from .tmp_dir import TmpDir
 from .transform import TF, Transform, apply_transform
 from .utils import PathLike, to_path
@@ -62,6 +65,13 @@ class Pferd(Location):
         for transformable in transformables:
             LOGGER.info(transformable.path)
 
+    @staticmethod
+    def _get_authenticator(
+            username: Optional[str], password: Optional[str]
+    ) -> KitShibbolethAuthenticator:
+        inner_auth = UserPassAuthenticator("ILIAS - Pferd.py", username, password)
+        return KitShibbolethAuthenticator(inner_auth)
+
     def _ilias(
             self,
             target: PathLike,
@@ -74,12 +84,13 @@ class Pferd(Location):
             download_strategy: IliasDownloadStrategy,
             timeout: int,
             clean: bool = True,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
     ) -> Organizer:
         # pylint: disable=too-many-locals
         cookie_jar = CookieJar(to_path(cookies) if cookies else None)
         session = cookie_jar.create_session()
         tmp_dir = self._tmp_dir.new_subdir()
-        organizer = Organizer(self.resolve(to_path(target)))
+        organizer = Organizer(self.resolve(to_path(target)), file_conflict_resolver)
 
         crawler = IliasCrawler(base_url, session, authenticator, dir_filter)
         downloader = IliasDownloader(tmp_dir, organizer, session,
@@ -115,6 +126,7 @@ class Pferd(Location):
             download_strategy: IliasDownloadStrategy = download_modified_or_new,
             clean: bool = True,
             timeout: int = 5,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
     ) -> Organizer:
         """
         Synchronizes a folder with the ILIAS instance of the KIT.
@@ -142,9 +154,11 @@ class Pferd(Location):
             clean {bool} -- Whether to clean up when the method finishes.
             timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
                 requests bug.
+            file_conflict_resolver {FileConflictResolver} -- A function specifying how to deal
+                with overwriting or deleting files. The default always asks the user.
         """
         # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
+        authenticator = Pferd._get_authenticator(username=username, password=password)
         PRETTY.starting_synchronizer(target, "ILIAS", course_id)
 
         organizer = self._ilias(
@@ -157,7 +171,8 @@ class Pferd(Location):
             transform=transform,
             download_strategy=download_strategy,
             clean=clean,
-            timeout=timeout
+            timeout=timeout,
+            file_conflict_resolver=file_conflict_resolver
         )
 
         self._download_summary.merge(organizer.download_summary)
@@ -182,6 +197,7 @@ class Pferd(Location):
             download_strategy: IliasDownloadStrategy = download_modified_or_new,
             clean: bool = True,
             timeout: int = 5,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
     ) -> Organizer:
         """
         Synchronizes a folder with the ILIAS instance of the KIT. This method will crawl the ILIAS
@@ -208,9 +224,11 @@ class Pferd(Location):
             clean {bool} -- Whether to clean up when the method finishes.
             timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
                 requests bug.
+            file_conflict_resolver {FileConflictResolver} -- A function specifying how to deal
+                with overwriting or deleting files. The default always asks the user.
         """
         # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
+        authenticator = Pferd._get_authenticator(username, password)
         PRETTY.starting_synchronizer(target, "ILIAS", "Personal Desktop")
 
         organizer = self._ilias(
@@ -223,7 +241,8 @@ class Pferd(Location):
             transform=transform,
             download_strategy=download_strategy,
             clean=clean,
-            timeout=timeout
+            timeout=timeout,
+            file_conflict_resolver=file_conflict_resolver
         )
 
         self._download_summary.merge(organizer.download_summary)
@@ -243,6 +262,7 @@ class Pferd(Location):
             download_strategy: IliasDownloadStrategy = download_modified_or_new,
             clean: bool = True,
             timeout: int = 5,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
     ) -> Organizer:
         """
         Synchronizes a folder with a given folder on the ILIAS instance of the KIT.
@@ -269,9 +289,11 @@ class Pferd(Location):
             clean {bool} -- Whether to clean up when the method finishes.
             timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
                 requests bug.
+            file_conflict_resolver {FileConflictResolver} -- A function specifying how to deal
+                with overwriting or deleting files. The default always asks the user.
         """
         # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
+        authenticator = Pferd._get_authenticator(username=username, password=password)
         PRETTY.starting_synchronizer(target, "ILIAS", "An ILIAS element by url")
 
         if not full_url.startswith("https://ilias.studium.kit.edu"):
@@ -287,13 +309,71 @@ class Pferd(Location):
             transform=transform,
             download_strategy=download_strategy,
             clean=clean,
-            timeout=timeout
+            timeout=timeout,
+            file_conflict_resolver=file_conflict_resolver
         )
 
         self._download_summary.merge(organizer.download_summary)
 
         return organizer
 
+    @swallow_and_print_errors
+    def ipd_kit(
+            self,
+            target: Union[PathLike, Organizer],
+            url: str,
+            transform: Transform = lambda x: x,
+            download_strategy: IpdDownloadStrategy = ipd_download_new_or_modified,
+            clean: bool = True,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
+    ) -> Organizer:
+        """
+        Synchronizes a folder with a DIVA playlist.
+
+        Arguments:
+            target {Union[PathLike, Organizer]} -- The organizer / target folder to use.
+            url {str} -- the url to the page
+
+        Keyword Arguments:
+            transform {Transform} -- A transformation function for the output paths. Return None
+                to ignore a file. (default: {lambdax:x})
+            download_strategy {DivaDownloadStrategy} -- A function to determine which files need to
+                be downloaded. Can save bandwidth and reduce the number of requests.
+                (default: {diva_download_new})
+            clean {bool} -- Whether to clean up when the method finishes.
+            file_conflict_resolver {FileConflictResolver} -- A function specifying how to deal
+                with overwriting or deleting files. The default always asks the user.
+        """
+        tmp_dir = self._tmp_dir.new_subdir()
+
+        if target is None:
+            PRETTY.starting_synchronizer("None", "IPD", url)
+            raise FatalException("Got 'None' as target directory, aborting")
+
+        if isinstance(target, Organizer):
+            organizer = target
+        else:
+            organizer = Organizer(self.resolve(to_path(target)), file_conflict_resolver)
+
+        PRETTY.starting_synchronizer(organizer.path, "IPD", url)
+
+        elements: List[IpdDownloadInfo] = IpdCrawler(url).crawl()
+        transformed = apply_transform(transform, elements)
+
+        if self._test_run:
+            self._print_transformables(transformed)
+            return organizer
+
+        downloader = IpdDownloader(tmp_dir=tmp_dir, organizer=organizer, strategy=download_strategy)
+        downloader.download_all(transformed)
+
+        if clean:
+            organizer.cleanup()
+
+        self._download_summary.merge(organizer.download_summary)
+
+        return organizer
+
     @swallow_and_print_errors
     def diva_kit(
             self,
@@ -301,7 +381,8 @@ class Pferd(Location):
             playlist_location: str,
             transform: Transform = lambda x: x,
             download_strategy: DivaDownloadStrategy = diva_download_new,
-            clean: bool = True
+            clean: bool = True,
+            file_conflict_resolver: FileConflictResolver = resolve_prompt_user
     ) -> Organizer:
         """
         Synchronizes a folder with a DIVA playlist.
@@ -318,6 +399,8 @@ class Pferd(Location):
                 be downloaded. Can save bandwidth and reduce the number of requests.
                 (default: {diva_download_new})
             clean {bool} -- Whether to clean up when the method finishes.
+            file_conflict_resolver {FileConflictResolver} -- A function specifying how to deal
+                with overwriting or deleting files. The default always asks the user.
         """
         tmp_dir = self._tmp_dir.new_subdir()
 
@@ -333,7 +416,7 @@ class Pferd(Location):
         if isinstance(target, Organizer):
             organizer = target
         else:
-            organizer = Organizer(self.resolve(to_path(target)))
+            organizer = Organizer(self.resolve(to_path(target)), file_conflict_resolver)
 
         PRETTY.starting_synchronizer(organizer.path, "DIVA", playlist_id)
 
@@ -352,4 +435,6 @@ class Pferd(Location):
         if clean:
             organizer.cleanup()
 
+        self._download_summary.merge(organizer.download_summary)
+
         return organizer

PFERD/transform.py

@@ -5,6 +5,8 @@ only files whose names match a regex, or renaming files from one numbering
 scheme to another.
 """
 
+import os
+import re
 from dataclasses import dataclass
 from pathlib import PurePath
 from typing import Callable, List, Optional, TypeVar
@@ -45,7 +47,8 @@ def apply_transform(
 
 # Transform combinators
 
-keep = lambda path: path
+def keep(path: PurePath) -> Optional[PurePath]:
+    return path
 
 def attempt(*args: Transform) -> Transform:
     def inner(path: PurePath) -> Optional[PurePath]:
@@ -125,3 +128,15 @@ def re_rename(regex: Regex, target: str) -> Transform:
             return path.with_name(target.format(*groups))
         return None
     return inner
+
+
+def sanitize_windows_path(path: PurePath) -> Optional[PurePath]:
+    """
+    A small function to escape characters that are forbidden in windows path names.
+    This method is a no-op on other operating systems.
+    """
+    # Escape windows illegal path characters
+    if os.name == 'nt':
+        sanitized_parts = [re.sub(r'[<>:"/|?]', "_", x) for x in list(path.parts)]
+        return PurePath(*sanitized_parts)
+    return path

README.md

@@ -25,7 +25,8 @@ use, but doesn't expose all the configuration options and tweaks a full install
 does.
 
 1. Download the `sync_url` binary from the [latest release](https://github.com/Garmelon/PFERD/releases/latest).
-2. Run the binary in your terminal (`./sync_url` or `sync_url.exe` in the CMD) to see the help and use it. I'd recommend using the `--cookies` option.  
+2. Recognize that you most likely need to enclose the URL in `""` quotes to prevent your shell from interpreting `&` and other symbols
+3. Run the binary in your terminal (`./sync_url` or `sync_url.exe` in the CMD) to see the help and use it. I'd recommend using the `--cookies` option.  
   If you are on **Linux/Mac**, you need to *make the file executable* using `chmod +x <file>`.  
   If you are on **Mac**, you need to allow this unverified program to run (see e.g. [here](https://www.switchingtomac.com/tutorials/osx/how-to-run-unverified-apps-on-macos/))
 
@@ -36,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
 To install PFERD or update your installation to the latest version, run this
 wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.4.1
+$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
 ```
 
 The use of [venv] is recommended.
@@ -59,8 +60,8 @@ $ mkdir Vorlesungen
 $ cd Vorlesungen
 $ python3 -m venv .venv
 $ .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.4.1
+$ pip install git+https://github.com/Garmelon/PFERD@v2.5.0
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.4.1/example_config.py
+$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.0/example_config.py
 $ python3 example_config.py
 $ deactivate
 ```

mypy.ini

@@ -3,5 +3,5 @@ disallow_untyped_defs = True
 disallow_incomplete_defs = True
 no_implicit_optional = True
 
-[mypy-rich.*,bs4]
+[mypy-rich.*,bs4,keyring]
 ignore_missing_imports = True

requirements.txt

@@ -0,0 +1,4 @@
+requests>=2.21.0
+beautifulsoup4>=4.7.1
+rich>=2.1.0
+keyring>=21.5.0

setup.py

@@ -2,12 +2,13 @@ from setuptools import find_packages, setup
 
 setup(
     name="PFERD",
-    version="2.4.1",
+    version="2.5.0",
     packages=find_packages(),
     install_requires=[
         "requests>=2.21.0",
         "beautifulsoup4>=4.7.1",
-        "rich>=2.1.0"
+        "rich>=2.1.0",
+        "keyring>=21.5.0"
     ],
 )
 

sync_url.py

@@ -5,63 +5,155 @@ A simple script to download a course by name from ILIAS.
 """
 
 import argparse
-from pathlib import Path
+import logging
+import sys
+from pathlib import Path, PurePath
+from typing import Optional
 from urllib.parse import urlparse
 
 from PFERD import Pferd
+from PFERD.authenticators import KeyringAuthenticator, UserPassAuthenticator
 from PFERD.cookie_jar import CookieJar
 from PFERD.ilias import (IliasCrawler, IliasElementType,
                          KitShibbolethAuthenticator)
+from PFERD.logging import PrettyLogger, enable_logging
+from PFERD.organizer import (ConflictType, FileConflictResolution,
+                             FileConflictResolver, resolve_prompt_user)
+from PFERD.transform import sanitize_windows_path
 from PFERD.utils import to_path
 
+_LOGGER = logging.getLogger("sync_url")
+_PRETTY = PrettyLogger(_LOGGER)
+
+
+def _extract_credentials(file_path: Optional[str]) -> UserPassAuthenticator:
+    if not file_path:
+        return UserPassAuthenticator("KIT ILIAS Shibboleth", None, None)
+
+    if not Path(file_path).exists():
+        _PRETTY.error("Credential file does not exist")
+        sys.exit(1)
+
+    with open(file_path, "r") as file:
+        first_line = file.read().splitlines()[0]
+        read_name, *read_password = first_line.split(":", 1)
+
+        name = read_name if read_name else None
+        password = read_password[0] if read_password else None
+        return UserPassAuthenticator("KIT ILIAS Shibboleth", username=name, password=password)
+
+
+def _resolve_remote_first(_path: PurePath, _conflict: ConflictType) -> FileConflictResolution:
+    return FileConflictResolution.DESTROY_EXISTING
+
+
+def _resolve_local_first(_path: PurePath, _conflict: ConflictType) -> FileConflictResolution:
+    return FileConflictResolution.KEEP_EXISTING
+
+
+def _resolve_no_delete(_path: PurePath, conflict: ConflictType) -> FileConflictResolution:
+    # Update files
+    if conflict == ConflictType.FILE_OVERWRITTEN:
+        return FileConflictResolution.DESTROY_EXISTING
+    if conflict == ConflictType.MARKED_FILE_OVERWRITTEN:
+        return FileConflictResolution.DESTROY_EXISTING
+    # But do not delete them
+    return FileConflictResolution.KEEP_EXISTING
+
 
 def main() -> None:
+    enable_logging(name="sync_url")
+
     parser = argparse.ArgumentParser()
     parser.add_argument("--test-run", action="store_true")
     parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in")
+    parser.add_argument('-u', '--username', nargs='?', default=None, help="Username for Ilias")
+    parser.add_argument('-p', '--password', nargs='?', default=None, help="Password for Ilias")
+    parser.add_argument('--credential-file', nargs='?', default=None,
+                        help="Path to a file containing credentials for Ilias. The file must have "
+                        "one line in the following format: '<user>:<password>'")
+    parser.add_argument("-k", "--keyring", action="store_true",
+                        help="Use the system keyring service for authentication")
     parser.add_argument('--no-videos', nargs='?', default=None, help="Don't download videos")
+    parser.add_argument('--local-first', action="store_true",
+                        help="Don't prompt for confirmation, keep existing files")
+    parser.add_argument('--remote-first', action="store_true",
+                        help="Don't prompt for confirmation, delete and overwrite local files")
+    parser.add_argument('--no-delete', action="store_true",
+                        help="Don't prompt for confirmation, overwrite local files, don't delete")
     parser.add_argument('url', help="URL to the course page")
     parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into")
     args = parser.parse_args()
 
-    url = urlparse(args.url)
-
     cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None)
     session = cookie_jar.create_session()
-    authenticator = KitShibbolethAuthenticator()
+
+    if args.keyring:
+        if not args.username:
+            _PRETTY.error("Keyring auth selected but no --username passed!")
+            return
+        inner_auth: UserPassAuthenticator = KeyringAuthenticator(
+            "KIT ILIAS Shibboleth", username=args.username, password=args.password
+        )
+    else:
+        inner_auth = _extract_credentials(args.credential_file)
+
+    username, password = inner_auth.get_credentials()
+    authenticator = KitShibbolethAuthenticator(inner_auth)
+
+    url = urlparse(args.url)
+
     crawler = IliasCrawler(url.scheme + '://' + url.netloc, session,
                            authenticator, lambda x, y: True)
 
     cookie_jar.load_cookies()
 
-    if args.folder is not None:
+    if args.folder is None:
-        folder = args.folder
+        element_name = crawler.find_element_name(args.url)
-        # Initialize pferd at the *parent of the passed folder*
+        if not element_name:
-        # This is needed so Pferd's internal protections against escaping the working directory
+            print("Error, could not get element name. Please specify a folder yourself.")
-        # do not trigger (e.g. if somebody names a file in ILIAS '../../bad thing.txt')
+            return
-        pferd = Pferd(Path(Path(__file__).parent, folder).parent, test_run=args.test_run)
+        folder = Path(element_name)
-    else:
-        # fetch course name from ilias
-        folder = crawler.find_element_name(args.url)
         cookie_jar.save_cookies()
+    else:
+        folder = Path(args.folder)
 
-        # Initialize pferd at the location of the script
+    # files may not escape the pferd_root with relative paths
-        pferd = Pferd(Path(__file__).parent, test_run=args.test_run)
+    # note: Path(Path.cwd, Path(folder)) == Path(folder) if it is an absolute path
+    pferd_root = Path(Path.cwd(), Path(folder)).parent
+    target = folder.name
+    pferd = Pferd(pferd_root, test_run=args.test_run)
 
     def dir_filter(_: Path, element: IliasElementType) -> bool:
         if args.no_videos:
             return element not in [IliasElementType.VIDEO_FILE, IliasElementType.VIDEO_FOLDER]
         return True
 
+    if args.local_first:
+        file_confilict_resolver: FileConflictResolver = _resolve_local_first
+    elif args.no_delete:
+        file_confilict_resolver = _resolve_no_delete
+    elif args.remote_first:
+        file_confilict_resolver = _resolve_remote_first
+    else:
+        file_confilict_resolver = resolve_prompt_user
+
     pferd.enable_logging()
+
     # fetch
     pferd.ilias_kit_folder(
-        target=folder,
+        target=target,
         full_url=args.url,
         cookies=args.cookies,
-        dir_filter=dir_filter
+        dir_filter=dir_filter,
+        username=username,
+        password=password,
+        file_conflict_resolver=file_confilict_resolver,
+        transform=sanitize_windows_path
     )
 
+    pferd.print_summary()
+
 
 if __name__ == "__main__":
     main()