Version 2.3.0

Some python versions just can't handle it *despite the documentation
stating they should*.

LICENSE

@@ -0,0 +1,18 @@
+Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

PFERD/cookie_jar.py

@@ -22,7 +22,7 @@ class CookieJar:
         if cookie_file is None:
             self._cookies = LWPCookieJar()
         else:
-            self._cookies = LWPCookieJar(cookie_file)
+            self._cookies = LWPCookieJar(str(cookie_file.resolve()))
 
     @property
     def cookies(self) -> LWPCookieJar:

PFERD/ilias/authenticators.py

@@ -67,7 +67,7 @@ class KitShibbolethAuthenticator(IliasAuthenticator):
         while not self._login_successful(soup):
             # Searching the form here so that this fails before asking for
             # credentials rather than after asking.
-            form = soup.find("form", {"class": "form2", "method": "post"})
+            form = soup.find("form", {"class": "full content", "method": "post"})
             action = form["action"]
 
             # Equivalent: Enter credentials in

PFERD/ilias/crawler.py

@@ -38,6 +38,12 @@ class IliasElementType(Enum):
     FORUM = "FORUM"
     EXTERNAL_LINK = "EXTERNAL_LINK"
 
+    def is_folder(self) -> bool:
+        """
+        Returns whether this type is some kind of folder.
+        """
+        return "FOLDER" in str(self.name)
+
 
 IliasDirectoryFilter = Callable[[Path, IliasElementType], bool]
 
@@ -167,7 +173,7 @@ class IliasCrawler:
                 PRETTY.not_searching(entry.path, "forum")
                 continue
 
-            if not self.dir_filter(entry.path, entry.entry_type):
+            if entry.entry_type.is_folder() and not self.dir_filter(entry.path, entry.entry_type):
                 PRETTY.not_searching(entry.path, "user filter")
                 continue
 

PFERD/ilias/downloader.py

@@ -2,6 +2,8 @@
 
 import datetime
 import logging
+import math
+import os
 from pathlib import Path, PurePath
 from typing import Callable, List, Optional, Union
 
@@ -82,9 +84,13 @@ class IliasDownloader:
             session: requests.Session,
             authenticator: IliasAuthenticator,
             strategy: IliasDownloadStrategy,
+            timeout: int = 5
     ):
         """
         Create a new IliasDownloader.
+
+        The timeout applies to the download request only, as bwcloud uses IPv6
+        and requests has a problem with that: https://github.com/psf/requests/issues/5522
         """
 
         self._tmp_dir = tmp_dir
@@ -92,6 +98,7 @@ class IliasDownloader:
         self._session = session
         self._authenticator = authenticator
         self._strategy = strategy
+        self._timeout = timeout
 
     def download_all(self, infos: List[IliasDownloadInfo]) -> None:
         """
@@ -119,7 +126,15 @@ class IliasDownloader:
             LOGGER.info("Retrying download: %r", info)
             self._authenticator.authenticate(self._session)
 
-        self._organizer.accept_file(tmp_file, info.path)
+        dst_path = self._organizer.accept_file(tmp_file, info.path)
+        if dst_path and info.modification_date:
+            os.utime(
+                dst_path,
+                times=(
+                    math.ceil(info.modification_date.timestamp()),
+                    math.ceil(info.modification_date.timestamp())
+                )
+            )
 
     def _try_download(self, info: IliasDownloadInfo, target: Path) -> bool:
         url = info.url()
@@ -127,10 +142,11 @@ class IliasDownloader:
             PRETTY.warning(f"Could not download {str(info.path)!r} as I got no URL :/")
             return True
 
-        with self._session.get(url, stream=True) as response:
+        with self._session.get(url, stream=True, timeout=self._timeout) as response:
             content_type = response.headers["content-type"]
+            has_content_disposition = "content-disposition" in response.headers
 
-            if content_type.startswith("text/html"):
+            if content_type.startswith("text/html") and not has_content_disposition:
                 if self._is_logged_in(soupify(response)):
                     raise ContentTypeException("Attempting to download a web page, not a file")
 

PFERD/organizer.py

@@ -5,9 +5,10 @@ A organizer is bound to a single directory.
 
 import filecmp
 import logging
+import os
 import shutil
 from pathlib import Path, PurePath
-from typing import List, Set
+from typing import List, Optional, Set
 
 from .download_summary import DownloadSummary
 from .location import Location
@@ -35,10 +36,26 @@ class Organizer(Location):
 
         self.download_summary = DownloadSummary()
 
-    def accept_file(self, src: Path, dst: PurePath) -> None:
-        """Move a file to this organizer and mark it."""
-        src_absolute = src.resolve()
-        dst_absolute = self.resolve(dst)
+    def accept_file(self, src: Path, dst: PurePath) -> Optional[Path]:
+        """
+        Move a file to this organizer and mark it.
+
+        Returns the path the file was moved to, to allow the caller to adjust the metadata.
+        As you might still need to adjust the metadata when the file was identical
+        (e.g. update the timestamp), the path is also returned in this case.
+        In all other cases (ignored, not overwritten, etc.) this method returns None.
+        """
+        # Windows limits the path length to 260 for *some* historical reason
+        # If you want longer paths, you will have to add the "\\?\" prefix in front of
+        # your path...
+        # See:
+        # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
+        if os.name == 'nt':
+            src_absolute = Path("\\\\?\\" + str(src.resolve()))
+            dst_absolute = Path("\\\\?\\" + str(self.resolve(dst)))
+        else:
+            src_absolute = src.resolve()
+            dst_absolute = self.resolve(dst)
 
         if not src_absolute.exists():
             raise FileAcceptException("Source file does not exist")
@@ -52,7 +69,7 @@ class Organizer(Location):
             PRETTY.warning(f"File {str(dst_absolute)!r} was already written!")
             if not prompt_yes_no(f"Overwrite file?", default=False):
                 PRETTY.ignored_file(dst_absolute, "file was written previously")
-                return
+                return None
 
         # Destination file is directory
         if dst_absolute.exists() and dst_absolute.is_dir():
@@ -60,7 +77,7 @@ class Organizer(Location):
                 shutil.rmtree(dst_absolute)
             else:
                 PRETTY.warning(f"Could not add file {str(dst_absolute)!r}")
-                return
+                return None
 
         # Destination file exists
         if dst_absolute.exists() and dst_absolute.is_file():
@@ -68,9 +85,7 @@ class Organizer(Location):
                 # Bail out, nothing more to do
                 PRETTY.ignored_file(dst_absolute, "same file contents")
                 self.mark(dst)
-                # Touch it to update the timestamp
-                dst_absolute.touch()
-                return
+                return dst_absolute
 
             self.download_summary.add_modified_file(dst_absolute)
             PRETTY.modified_file(dst_absolute)
@@ -87,6 +102,8 @@ class Organizer(Location):
 
         self.mark(dst)
 
+        return dst_absolute
+
     def mark(self, path: PurePath) -> None:
         """Mark a file as used so it will not get cleaned up."""
         absolute_path = self.resolve(path)

PFERD/pferd.py

@@ -72,7 +72,8 @@ class Pferd(Location):
             dir_filter: IliasDirectoryFilter,
             transform: Transform,
             download_strategy: IliasDownloadStrategy,
-            clean: bool = True
+            timeout: int,
+            clean: bool = True,
     ) -> Organizer:
         # pylint: disable=too-many-locals
         cookie_jar = CookieJar(to_path(cookies) if cookies else None)
@@ -81,7 +82,8 @@ class Pferd(Location):
         organizer = Organizer(self.resolve(to_path(target)))
 
         crawler = IliasCrawler(base_url, session, authenticator, dir_filter)
-        downloader = IliasDownloader(tmp_dir, organizer, session, authenticator, download_strategy)
+        downloader = IliasDownloader(tmp_dir, organizer, session,
+                                     authenticator, download_strategy, timeout)
 
         cookie_jar.load_cookies()
         info = crawl_function(crawler)
@@ -112,6 +114,7 @@ class Pferd(Location):
             password: Optional[str] = None,
             download_strategy: IliasDownloadStrategy = download_modified_or_new,
             clean: bool = True,
+            timeout: int = 5,
     ) -> Organizer:
         """
         Synchronizes a folder with the ILIAS instance of the KIT.
@@ -137,6 +140,8 @@ class Pferd(Location):
                 be downloaded. Can save bandwidth and reduce the number of requests.
                 (default: {download_modified_or_new})
             clean {bool} -- Whether to clean up when the method finishes.
+            timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
+                requests bug.
         """
         # This authenticator only works with the KIT ilias instance.
         authenticator = KitShibbolethAuthenticator(username=username, password=password)
@@ -152,6 +157,7 @@ class Pferd(Location):
             transform=transform,
             download_strategy=download_strategy,
             clean=clean,
+            timeout=timeout
         )
 
         self._download_summary.merge(organizer.download_summary)
@@ -175,6 +181,7 @@ class Pferd(Location):
             password: Optional[str] = None,
             download_strategy: IliasDownloadStrategy = download_modified_or_new,
             clean: bool = True,
+            timeout: int = 5,
     ) -> Organizer:
         """
         Synchronizes a folder with the ILIAS instance of the KIT. This method will crawl the ILIAS
@@ -199,11 +206,14 @@ class Pferd(Location):
                 be downloaded. Can save bandwidth and reduce the number of requests.
                 (default: {download_modified_or_new})
             clean {bool} -- Whether to clean up when the method finishes.
+            timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
+                requests bug.
         """
         # This authenticator only works with the KIT ilias instance.
         authenticator = KitShibbolethAuthenticator(username=username, password=password)
         PRETTY.starting_synchronizer(target, "ILIAS", "Personal Desktop")
-        return self._ilias(
+
+        organizer = self._ilias(
             target=target,
             base_url="https://ilias.studium.kit.edu/",
             crawl_function=lambda crawler: crawler.crawl_personal_desktop(),
@@ -213,8 +223,13 @@ class Pferd(Location):
             transform=transform,
             download_strategy=download_strategy,
             clean=clean,
+            timeout=timeout
         )
 
+        self._download_summary.merge(organizer.download_summary)
+
+        return organizer
+
     @swallow_and_print_errors
     def diva_kit(
             self,

README.md

@@ -2,43 +2,48 @@
 
 **P**rogramm zum **F**lotten, **E**infachen **R**unterladen von **D**ateien
 
+- [Installation](#installation)
+    - [Upgrading from 2.0.0 to 2.1.0+](#upgrading-from-200-to-210)
+- [Example setup](#example-setup)
+- [Usage](#usage)
+    - [General concepts](#general-concepts)
+    - [Constructing transforms](#constructing-transforms)
+        - [Transform creators](#transform-creators)
+        - [Transform combinators](#transform-combinators)
+    - [A short, but commented example](#a-short-but-commented-example)
+
 ## Installation
 
 Ensure that you have at least Python 3.8 installed.
 
 To install PFERD or update your installation to the latest version, run this
-wherever you want to install/have installed PFERD:
+wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.1.2
+$ pip install git+https://github.com/Garmelon/PFERD@v2.3.0
 ```
 
-The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
+The use of [venv] is recommended.
+
+[venv]: https://docs.python.org/3/library/venv.html
 
 ### Upgrading from 2.0.0 to 2.1.0+
 
-The `IliasDirectoryType` type was renamed to `IliasElementType` and is now far
-more detailed.  
-The new values are: REGULAR_FOLDER, VIDEO_FOLDER,
-EXERCISE_FOLDER, REGULAR_FILE, VIDEO_FILE, FORUM, EXTERNAL_LINK.  
-Forums and external links are skipped automatically if you use the `kit_ilias` helper.
+- The `IliasDirectoryType` type was renamed to `IliasElementType` and is now far more detailed.
+  The new values are: `REGULAR_FOLDER`, `VIDEO_FOLDER`, `EXERCISE_FOLDER`, `REGULAR_FILE`, `VIDEO_FILE`, `FORUM`, `EXTERNAL_LINK`.
+- Forums and external links are skipped automatically if you use the `kit_ilias` helper.
 
 ## Example setup
 
 In this example, `python3` refers to at least Python 3.8.
 
-If you just want to get started and crawl *your entire ILIAS Desktop* instead
-of a given set of courses, please replace `example_config.py` with
-`example_config_personal_desktop.py` in all of the instructions below (`curl` call and
-`python3` run command).
-
 A full example setup and initial use could look like:
 ```
 $ mkdir Vorlesungen
 $ cd Vorlesungen
 $ python3 -m venv .venv
 $ .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.1.2
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.1.2/example_config.py
+$ pip install git+https://github.com/Garmelon/PFERD@v2.3.0
+$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.3.0/example_config.py
 $ python3 example_config.py
 $ deactivate
 ```
@@ -51,50 +56,93 @@ $ python3 example_config.py
 $ deactivate
 ```
 
+If you just want to get started and crawl *your entire ILIAS Desktop* instead
+of a given set of courses, please replace `example_config.py` with
+`example_config_personal_desktop.py` in all of the instructions below (`curl` call and
+`python3` run command).
+
 ## Usage
 
+### General concepts
+
 A PFERD config is a normal python file that starts multiple *synchronizers*
 which do all the heavy lifting. While you can create and wire them up manually,
 you are encouraged to use the helper methods provided in `PFERD.Pferd`.
 
 The synchronizers take some input arguments specific to their service and a
-*transformer*. The transformer receives the computed path of an element in
-ILIAS and can return either an output path (so you can rename files or move
-them around as you wish) or `None` if you do not want to save the given file.
+*transform*. The transform receives the computed path of an element in ILIAS and
+can return either an output path (so you can rename files or move them around as
+you wish) or `None` if you do not want to save the given file.
 
 Additionally the ILIAS synchronizer allows you to define a *crawl filter*. This
-filter also receives the computed path as the input, but is only called or
-*directoties*. If you return `True`, the directory will be crawled and
+filter also receives the computed path as the input, but is only called for
+*directories*. If you return `True`, the directory will be crawled and
 searched. If you return `False` the directory will be ignored and nothing in it
-will be passed to the transformer.
+will be passed to the transform.
 
-In order to help you with writing your own transformers and filters, PFERD
-ships with a few powerful building blocks:
+### Constructing transforms
 
-| Method | Description |
-|--------|-------------|
-| `glob`   | Returns a transform that returns `None` if the glob does not match and the unmodified path otherwise. |
-| `predicate`   | Returns a transform that returns `None` if the predicate does not match the path and the unmodified path otherwise. |
-| `move_dir(source, target)`   | Returns a transform that moves all files from the `source` to the `target` dir. |
-| `move(source, target)`   | Returns a transform that moves the `source` file to `target`. |
-| `rename(old, new)`   | Renames a single file. |
-| `re_move(regex, sub)`   | Moves all files matching the given regular expression. The different captured groups are available under their index and can be used together with normal python format methods: `re_move(r"Blatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf"),`. |
-| `re_rename(old, new)`   | Same as `re_move` but operates on the path *names* instead of the full path. |
+While transforms are just normal python functions, writing them by hand can
+quickly become tedious. In order to help you with writing your own transforms
+and filters, PFERD defines a few useful transform creators and combinators in
+the `PFERD.transform` module:
 
-And PFERD also offers a few combinator functions:
+#### Transform creators
 
-* **`keep`**  
-  `keep` just returns the input path unchanged. It can be very useful as the
-  last argument in an `attempt` call, to leave everything not matching a rule
-  unchanged.
-* **`optionally(transformer)`**  
-  Wraps a given transformer and returns its result if it is not `None`.
+These methods let you create a few basic transform building blocks:
+
+- **`glob(glob)`**  
+  Creates a transform that returns the unchanged path if the glob matches the path and `None` otherwise.
+  See also [Path.match].  
+  Example: `glob("Übung/*.pdf")`
+- **`predicate(pred)`**  
+  Creates a transform that returns the unchanged path if `pred(path)` returns a truthy value.
+  Returns `None` otherwise.  
+  Example: `predicate(lambda path: len(path.parts) == 3)`
+- **`move_dir(source, target)`**  
+  Creates a transform that moves all files from the `source` to the `target` directory.  
+  Example: `move_dir("Übung/", "Blätter/")`
+- **`move(source, target)`**  
+  Creates a transform that moves the `source` file to `target`.  
+  Example: `move("Vorlesung/VL02_Automten.pdf", "Vorlesung/VL02_Automaten.pdf")`
+- **`rename(source, target)`**  
+  Creates a transform that renames all files named `source` to `target`.
+  This transform works on the file names, not paths, and thus works no matter where the file is located.  
+  Example: `rename("VL02_Automten.pdf", "VL02_Automaten.pdf")`
+- **`re_move(regex, target)`**  
+  Creates a transform that moves all files matching `regex` to `target`.
+  The transform `str.format` on the `target` string with the contents of the capturing groups before returning it.
+  The capturing groups can be accessed via their index.
+  See also [Match.group].  
+  Example: `re_move(r"Übung/Blatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf")`
+- **`re_rename(regex, target)`**  
+  Creates a transform that renames all files matching `regex` to `target`.
+  This transform works on the file names, not paths, and thus works no matter where the file is located.  
+  Example: `re_rename(r"VL(\d+)(.*)\.pdf", "Vorlesung_Nr_{1}__{2}.pdf")`
+
+All movement or rename transforms above return `None` if a file doesn't match
+their movement or renaming criteria. This enables them to be used as building
+blocks to build up more complex transforms.
+
+In addition, `PFERD.transform` also defines the `keep` transform which returns its input path unchanged.
+This behaviour can be very useful when creating more complex transforms.
+See below for example usage.
+
+[Path.match]: https://docs.python.org/3/library/pathlib.html#pathlib.Path.match
+[Match.group]: https://docs.python.org/3/library/re.html#re.Match.group
+
+#### Transform combinators
+
+These methods let you combine transforms into more complex transforms:
+
+- **`optionally(transform)`**  
+  Wraps a given transform and returns its result if it is not `None`.
   Otherwise returns the input path unchanged.
-* **`do(transformers)`**  
-  `do` accepts a series of transformers and applies them in the given order to
-  the result of the previous one. If any transformer returns `None`, do
-  short-circuits and also returns `None`. This can be used to perform multiple
-  renames in a row:
+  See below for example usage.
+* **`do(transforms)`**  
+  Accepts a series of transforms and applies them in the given order to the result of the previous one.
+  If any transform returns `None`, `do` short-circuits and also returns `None`.
+  This can be used to perform multiple renames in a row:
   ```py
   do(
       # Move them
@@ -103,13 +151,12 @@ And PFERD also offers a few combinator functions:
       optionally(re_rename("(.*).m4v.mp4", "{1}.mp4")),
       # Remove the 'dbs' prefix (if they have any)
       optionally(re_rename("(?i)dbs-(.+)", "{1}")),
-  ),
+  )
   ```
-* **`attempt(transformers)`**  
-  `attempt` applies the passed transformers in the given order until it finds
-  one that does not return `None`. If it does not find any, it returns `None`.
-  This can be used to give a list of possible transformations and it will
-  automatically pick the first one that fits:
+- **`attempt(transforms)`**  
+  Applies the passed transforms in the given order until it finds one that does not return `None`.
+  If it does not find any, it returns `None`.
+  This can be used to give a list of possible transformations and automatically pick the first one that fits:
   ```py
   attempt(
       # Move all videos. If a video is passed in, this `re_move` will succeed
@@ -122,17 +169,26 @@ And PFERD also offers a few combinator functions:
   )
   ```
 
-All of these combinators are used in the provided example config, if you want
-to see some more true-to-live usages.
+All of these combinators are used in the provided example configs, if you want
+to see some more real-life usages.
 
 ### A short, but commented example
 
 ```py
-def filter_course(path: PurePath) -> bool:
-    # Note that glob returns a Transformer
-    #  - a function from PurePath -> Optional[PurePath]
-    # So we need to apply the result of 'glob' to our input path.
-    # We need to crawl the 'Tutorien' folder as it contains the one we want.
+from pathlib import Path, PurePath
+from PFERD import Pferd
+from PFERD.ilias import IliasElementType
+from PFERD.transform import *
+
+# This filter will later be used by the ILIAS crawler to decide whether it
+# should crawl a directory (or directory-like structure).
+def filter_course(path: PurePath, type: IliasElementType) -> bool:
+    # Note that glob returns a Transform, which is a function from PurePath ->
+    # Optional[PurePath]. Because of this, we need to apply the result of
+    # 'glob' to our input path. The returned value will be truthy (a Path) if
+    # the transform succeeded, or `None` if it failed.
+
+    # We need to crawl the 'Tutorien' folder as it contains one that we want.
     if glob("Tutorien/")(path):
         return True
     # If we found 'Tutorium 10', keep it!
@@ -145,21 +201,35 @@ def filter_course(path: PurePath) -> bool:
     # All other dirs (including subdirs of 'Tutorium 10') should be searched :)
     return True
 
-enable_logging() # needed once before calling a Pferd method
-# Create a Pferd instance rooted in the same directory as the script file
-# This is not a test run, so files will be downloaded (default, can be omitted)
+
+# This transform will later be used to rename a few files. It can also be used
+# to ignore some files.
+transform_course = attempt(
+    # We don't care about the other tuts and would instead prefer a cleaner
+    # directory structure.
+    move_dir("Tutorien/Tutorium 10/", "Tutorium/"),
+    # We don't want to modify any other files, so we're going to keep them
+    # exactly as they are.
+    keep
+)
+
+# Enable and configure the text output. Needs to be called before calling any
+# other PFERD methods.
+Pferd.enable_logging()
+# Create a Pferd instance rooted in the same directory as the script file. This
+# is not a test run, so files will be downloaded (default, can be omitted).
 pferd = Pferd(Path(__file__).parent, test_run=False)
 
 # Use the ilias_kit helper to synchronize an ILIAS course
 pferd.ilias_kit(
-    # The folder all of the course's content should be placed in
-    Path("My cool course"),
+    # The directory that all of the downloaded files should be placed in
+    "My_cool_course/",
     # The course ID (found in the URL when on the course page in ILIAS)
     "course id",
-    # A path to a cookie jar. If you synchronize multiple ILIAS courses, setting this
-    # to a common value requires you to only login once.
+    # A path to a cookie jar. If you synchronize multiple ILIAS courses,
+    # setting this to a common value requires you to only log in once.
     cookies=Path("ilias_cookies.txt"),
-    # A transform to apply to all found paths
+    # A transform can rename, move or filter out certain files
     transform=transform_course,
     # A crawl filter limits what paths the cralwer searches
     dir_filter=filter_course,

setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="PFERD",
-    version="2.1.2",
+    version="2.3.0",
     packages=find_packages(),
     install_requires=[
         "requests>=2.21.0",