Fix video name deduplication

If set to `no`, PFERD won't print status or report messages for not deleted files

.github/workflows/build-and-release.yml

@@ -0,0 +1,78 @@
+name: build-and-release
+
+on: push
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python: ["3.9"]
+    steps:
+
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Set up project
+        if: matrix.os != 'windows-latest'
+        run: ./scripts/setup
+
+      - name: Set up project on windows
+        if: matrix.os == 'windows-latest'
+        # For some reason, `pip install --upgrade pip` doesn't work on
+        # 'windows-latest'. The installed pip version works fine however.
+        run: ./scripts/setup --no-pip
+
+      - name: Run checks
+        run: ./scripts/check
+
+      - name: Build
+        run: ./scripts/build
+
+      - name: Rename binary
+        # Glob in source location because on windows pyinstaller creates a file
+        # named "pferd.exe"
+        run: mv dist/pferd* dist/pferd-${{ matrix.os }}
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: Binaries
+          path: dist/pferd-${{ matrix.os }}
+
+  release:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: build
+    steps:
+
+      - name: Download binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: Binaries
+
+      - name: Rename binaries
+        run: |
+          mv pferd-ubuntu-latest pferd-linux
+          mv pferd-windows-latest pferd-windows.exe
+          mv pferd-macos-latest pferd-mac
+
+      - name: Create release
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: |
+            pferd-linux
+            pferd-windows.exe
+            pferd-mac

.github/workflows/package.yml

@@ -1,67 +0,0 @@
-name: Package Application with Pyinstaller
-
-on:
-  push:
-    branches:
-      - "*"
-    tags:
-      - "v*"
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-
-    - name: "Install dependencies"
-      run: "pip install setuptools pyinstaller rich requests beautifulsoup4 -f --upgrade"
-
-    - name: "Install sync_url.py"
-      run: "pyinstaller sync_url.py -F"
-
-    - uses: actions/upload-artifact@v2
-      with:
-        name: "Pferd Sync URL"
-        path: "dist/sync_url*"
-
-  release:
-    name: Release
-
-    needs: [build]
-    runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags/')
-
-    env:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-    steps:
-    - name: "Checkout"
-      uses: actions/checkout@v2
-
-    - name: "Download artifacts"
-      uses: actions/download-artifact@v2
-      with:
-        name: "Pferd Sync URL"
-
-    - name: "look at folder structure"
-      run: "ls -lah"
-
-    - name: "Create release"
-      uses: softprops/action-gh-release@v1
-
-    - name: "Upload release artifacts"
-      uses: softprops/action-gh-release@v1
-      with:
-        body: "Download sync_url (or sync_url.exe on Windows) and run it in the terminal or CMD."
-        files: |
-          sync_url
-          sync_url.exe

.gitignore

@@ -1,14 +1,11 @@
-__pycache__/
-.venv/
-venv/
-.idea/
-build/
 .mypy_cache/
-.tmp/
-.env
-.vscode
-ilias_cookies.txt
+/.venv/
+/PFERD.egg-info/
+__pycache__/
+/.vscode/
+/.idea/
 
-# PyInstaller
-sync_url.spec
-dist/
+# pyinstaller
+/pferd.spec
+/build/
+/dist/

CHANGELOG.md

@@ -0,0 +1,211 @@
+# Changelog
+
+All notable changes to this project will be documented in this file. The format
+is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+This project has its own custom versioning scheme. Version numbers consist of
+three parts (e. g. `3.1.5`).
+- The first number is increased on major rewrites or changes. What classifies as
+  a major change is up to the maintainers. This is pretty rare and a PFERD
+  version 4 should hopefully not be necessary.
+- The second number is increased on backwards-incompatible changes in behaviour.
+  This refers to any change that would make an existing setup behave differently
+  (e. g. renaming options or changing crawler behaviour). If this number is
+  increased, it may be necessary for you to adapt your own setup.
+- The third number is increased on backwards-compatible changes (e. g. adding
+  new options or commands, changing documentation, fixing bugs). Updates that
+  only increase this number should be safe and not require manual intervention.
+
+We will try to correctly classify changes as backwards-compatible or
+backwards-incompatible, but may occasionally make mistakes or stumble across
+ambiguous situations.
+
+## Unreleased
+
+### Fixed
+- Video name deduplication
+
+## 3.5.0 - 2023-09-13
+
+### Added
+- `no-delete-prompt-override` conflict resolution strategy
+- Support for ILIAS learning modules
+- `show_not_deleted` option to stop printing the "Not Deleted" status or report
+  message. This combines nicely with the `no-delete-prompt-override` strategy,
+  causing PFERD to mostly ignore local-only files.
+- Support for mediacast video listings
+- Crawling of files in info tab
+
+### Changed
+- Remove size suffix for files in content pages
+
+### Fixed
+- Crawling of courses with the timeline view as the default tab
+- Crawling of file and custom opencast cards
+- Crawling of button cards without descriptions
+- Abort crawling when encountering an unexpected ilias root page redirect
+- Sanitize ascii control characters on Windows
+- Crawling of paginated past meetings
+- Ignore SCORM learning modules
+
+## 3.4.3 - 2022-11-29
+
+### Added
+- Missing documentation for `forums` option
+
+### Changed
+- Clear up error message shown when multiple paths are found to an element
+
+### Fixed
+- IPD crawler unnecessarily appending trailing slashes
+- Crawling opencast when ILIAS is set to English
+
+## 3.4.2 - 2022-10-26
+
+### Added
+- Recognize and crawl content pages in cards
+- Recognize and ignore surveys
+
+### Fixed
+- Forum crawling crashing when a thread has no messages at all
+- Forum crawling crashing when a forum has no threads at all
+- Ilias login failing in some cases
+- Crawling of paginated future meetings
+- IPD crawler handling of URLs without trailing slash
+
+## 3.4.1 - 2022-08-17
+
+### Added
+- Download of page descriptions
+- Forum download support
+- `pass` authenticator
+
+### Changed
+- Add `cpp` extension to default `link_regex` of IPD crawler
+- Mention hrefs in IPD crawler's `--explain` output for users of `link_regex` option
+- Simplify default IPD crawler `link_regex`
+
+### Fixed
+- IPD crawler crashes on some sites
+- Meeting name normalization for yesterday, today and tomorrow
+- Crawling of meeting file previews
+- Login with new login button html layout
+- Descriptions for courses are now placed in the correct subfolder when
+  downloading the whole desktop
+
+## 3.4.0 - 2022-05-01
+
+### Added
+- Message when Shibboleth entitlements need to be manually reviewed
+- Links to unofficial packages and repology in the readme
+
+### Changed
+- Increase minimum supported Python version to 3.9
+- Support video listings with more columns
+- Use UTF-8 when reading/writing the config file
+
+### Fixed
+- Crash during authentication when the Shibboleth session is still valid
+
+## 3.3.1 - 2022-01-15
+
+### Fixed
+- ILIAS login
+- Local video cache if `windows_paths` is enabled
+
+## 3.3.0 - 2022-01-09
+
+### Added
+- A KIT IPD crawler
+- Support for ILIAS cards
+- (Rudimentary) support for content pages
+- Support for multi-stream videos
+- Support for ILIAS 7
+
+### Removed
+- [Interpolation](https://docs.python.org/3/library/configparser.html#interpolation-of-values) in config file
+
+### Fixed
+- Crawling of recursive courses
+- Crawling files directly placed on the personal desktop
+- Ignore timestamps at the unix epoch as they crash on windows
+
+## 3.2.0 - 2021-08-04
+
+### Added
+- `--skip` command line option
+- Support for ILIAS booking objects
+
+### Changed
+- Using multiple path segments on left side of `-name->` now results in an
+  error. This was already forbidden by the documentation but silently accepted
+  by PFERD.
+- More consistent path printing in some `--explain` messages
+
+### Fixed
+- Nondeterministic name deduplication due to ILIAS reordering elements
+- More exceptions are handled properly
+
+## 3.1.0 - 2021-06-13
+
+If your config file doesn't do weird things with transforms, it should continue
+to work. If your `-re->` arrows behave weirdly, try replacing them with
+`-exact-re->` arrows. If you're on Windows, you might need to switch from `\`
+path separators to `/` in your regex rules.
+
+### Added
+- `skip` option for crawlers
+- Rules with `>>` instead of `>` as arrow head
+- `-exact-re->` arrow (behaves like `-re->` did previously)
+
+### Changed
+- The `-re->` arrow can now rename directories (like `-->`)
+- Use `/` instead of `\` as path separator for (regex) rules on Windows
+- Use the label to the left for exercises instead of the button name to
+  determine the folder name
+
+### Fixed
+- Video pagination handling in ILIAS crawler
+
+## 3.0.1 - 2021-06-01
+
+### Added
+- `credential-file` authenticator
+- `--credential-file` option for `kit-ilias-web` command
+- Warning if using concurrent tasks with `kit-ilias-web`
+
+### Changed
+- Cookies are now stored in a text-based format
+
+### Fixed
+- Date parsing now also works correctly in non-group exercises
+
+## 3.0.0 - 2021-05-31
+
+### Added
+- Proper config files
+- Concurrent crawling
+- Crawl external ILIAS links
+- Crawl uploaded exercise solutions
+- Explain what PFERD is doing and why (`--explain`)
+- More control over output (`--status`, `--report`)
+- Debug transform rules with `--debug-transforms`
+- Print report after exiting via Ctrl+C
+- Store crawler reports in `.report` JSON file
+- Extensive config file documentation (`CONFIG.md`)
+- Documentation for developers (`DEV.md`)
+- This changelog
+
+### Changed
+- Rewrote almost everything
+- Better error messages
+- Redesigned CLI
+- Redesigned transform rules
+- ILIAS crawling logic (paths may be different)
+- Better support for weird paths on Windows
+- Set user agent (`PFERD/<version>`)
+
+### Removed
+- Backwards compatibility with 2.x
+- Python files as config files
+- Some types of crawlers

CONFIG.md

@@ -0,0 +1,479 @@
+# Config file format
+
+A config file consists of sections. A section begins with a `[section]` header,
+which is followed by a list of `key = value` pairs. Comments must be on their
+own line and start with `#`. Multiline values must be indented beyond their key.
+Boolean values can be `yes` or `no`. For more details and some examples on the
+format, see the [configparser documentation][1] ([interpolation][2] is
+disabled).
+
+[1]: <https://docs.python.org/3/library/configparser.html#supported-ini-file-structure> "Supported INI File Structure"
+[2]: <https://docs.python.org/3/library/configparser.html#interpolation-of-values> "Interpolation of values"
+
+## The `DEFAULT` section
+
+This section contains global configuration values. It can also be used to set
+default values for the other sections.
+
+- `working_dir`: The directory PFERD operates in. Set to an absolute path to
+  make PFERD operate the same regardless of where it is executed from. All other
+  paths in the config file are interpreted relative to this path. If this path
+  is relative, it is interpreted relative to the script's working dir. `~` is
+  expanded to the current user's home directory. (Default: `.`)
+- `explain`: Whether PFERD should log and explain its actions and decisions in
+  detail. (Default: `no`)
+- `status`: Whether PFERD should print status updates (like `Crawled ...`,
+  `Added ...`) while running a crawler. (Default: `yes`)
+- `report`: Whether PFERD should print a report of added, changed and deleted
+   local files for all crawlers before exiting. (Default: `yes`)
+- `show_not_deleted`: Whether PFERD should print messages in status and report
+   when a local-only file wasn't deleted. Combines nicely with the
+   `no-delete-prompt-override` conflict resolution strategy.
+- `share_cookies`: Whether crawlers should share cookies where applicable. For
+  example, some crawlers share cookies if they crawl the same website using the
+  same account. (Default: `yes`)
+
+## The `crawl:*` sections
+
+Sections whose names start with `crawl:` are used to configure crawlers. The
+rest of the section name specifies the name of the crawler.
+
+A crawler synchronizes a remote resource to a local directory. There are
+different types of crawlers for different kinds of resources, e.g. ILIAS
+courses or lecture websites.
+
+Each crawl section represents an instance of a specific type of crawler. The
+`type` option is used to specify the crawler type. The crawler's name is usually
+used as the output directory. New crawlers can be created simply by adding a new
+crawl section to the config file.
+
+Depending on a crawler's type, it may have different options. For more details,
+see the type's [documentation](#crawler-types) below. The following options are
+common to all crawlers:
+
+- `type`: The available types are specified in [this section](#crawler-types).
+- `skip`: Whether the crawler should be skipped during normal execution. The
+  crawler can still be executed manually using the `--crawler` or `-C` flags.
+  (Default: `no`)
+- `output_dir`: The directory the crawler synchronizes files to. A crawler will
+  never place any files outside this directory. (Default: the crawler's name)
+- `redownload`: When to download a file that is already present locally.
+  (Default: `never-smart`)
+    - `never`: If a file is present locally, it is not downloaded again.
+    - `never-smart`: Like `never`, but PFERD tries to detect if an already
+      downloaded files has changed via some (unreliable) heuristics.
+    - `always`: All files are always downloaded, regardless of whether they are
+      already present locally.
+    - `always-smart`: Like `always`, but PFERD tries to avoid unnecessary
+      downloads via some (unreliable) heuristics.
+- `on_conflict`: What to do when the local and remote versions of a file or
+  directory differ, including when a file is replaced by a directory or a
+  directory by a file. (Default: `prompt`)
+    - `prompt`: Always ask the user before overwriting or deleting local files
+      and directories.
+    - `local-first`: Always keep the local file or directory. Equivalent to
+      using `prompt` and always choosing "no". Implies that `redownload` is set
+      to `never`.
+    - `remote-first`: Always keep the remote file or directory. Equivalent to
+      using `prompt` and always choosing "yes".
+    - `no-delete`: Never delete local files, but overwrite local files if the
+      remote file is different.
+    - `no-delete-prompt-overwrite`: Never delete local files, but prompt to
+      overwrite local files if the remote file is different. Combines nicely
+      with the `show_not_deleted` option.
+- `transform`: Rules for renaming and excluding certain files and directories.
+  For more details, see [this section](#transformation-rules). (Default: empty)
+- `tasks`: The maximum number of concurrent tasks (such as crawling or
+  downloading). (Default: `1`)
+- `downloads`: How many of those tasks can be download tasks at the same time.
+  Must not be greater than `tasks`. (Default: Same as `tasks`)
+- `task_delay`: Time (in seconds) that the crawler should wait between
+  subsequent tasks. Can be used as a sort of rate limit to avoid unnecessary
+  load for the crawl target. (Default: `0.0`)
+- `windows_paths`: Whether PFERD should find alternative names for paths that
+  are invalid on Windows. (Default: `yes` on Windows, `no` otherwise)
+
+Some crawlers may also require credentials for authentication. To configure how
+the crawler obtains its credentials, the `auth` option is used. It is set to the
+full name of an auth section (including the `auth:` prefix).
+
+Here is a simple example:
+
+```ini
+[auth:example]
+type = simple
+username = foo
+password = bar
+
+[crawl:something]
+type = some-complex-crawler
+auth = auth:example
+on_conflict = no-delete
+tasks = 3
+```
+
+## The `auth:*` sections
+
+Sections whose names start with `auth:` are used to configure authenticators. An
+authenticator provides a username and a password to one or more crawlers.
+
+Authenticators work similar to crawlers: A section represents an authenticator
+instance whose name is the rest of the section name. The type is specified by
+the `type` option.
+
+Depending on an authenticator's type, it may have different options. For more
+details, see the type's [documentation](#authenticator-types) below. The only
+option common to all authenticators is `type`:
+
+- `type`: The types are specified in [this section](#authenticator-types).
+
+## Crawler types
+
+### The `local` crawler
+
+This crawler crawls a local directory. It is really simple and mostly useful for
+testing different setups. The various delay options are meant to make the
+crawler simulate a slower, network-based crawler.
+
+- `target`: Path to the local directory to crawl. (Required)
+- `crawl_delay`: Artificial delay (in seconds) to simulate for crawl requests.
+  (Default: `0.0`)
+- `download_delay`: Artificial delay (in seconds) to simulate for download
+  requests. (Default: `0.0`)
+- `download_speed`: Download speed (in bytes per second) to simulate. (Optional)
+
+### The `kit-ipd` crawler
+
+This crawler crawls a KIT-IPD page by url. The root page can be crawled from
+outside the KIT network so you will be informed about any new/deleted files,
+but downloading files requires you to be within. Adding a show delay between
+requests is likely a good idea.
+
+- `target`: URL to a KIT-IPD page
+- `link_regex`: A regex that is matched against the `href` part of links. If it
+  matches, the given link is downloaded as a file. This is used to extract
+  files from KIT-IPD pages. (Default: `^.*?[^/]+\.(pdf|zip|c|cpp|java)$`)
+
+### The `kit-ilias-web` crawler
+
+This crawler crawls the KIT ILIAS instance.
+
+ILIAS is not great at handling too many concurrent requests. To avoid
+unnecessary load, please limit `tasks` to `1`.
+
+There is a spike in ILIAS usage at the beginning of lectures, so please don't
+run PFERD during those times.
+
+If you're automatically running PFERD periodically (e. g. via cron or a systemd
+timer), please randomize the start time or at least don't use the full hour. For
+systemd timers, this can be accomplished using the `RandomizedDelaySec` option.
+Also, please schedule the script to run in periods of low activity. Running the
+script once per day should be fine.
+
+- `target`: The ILIAS element to crawl. (Required)
+    - `desktop`: Crawl your personal desktop
+    - `<course id>`: Crawl the course with the given id
+    - `<url>`: Crawl a given element by URL (preferably the permanent URL linked
+      at the bottom of its ILIAS page)
+- `auth`: Name of auth section to use for login. (Required)
+- `tfa_auth`: Name of auth section to use for two-factor authentication. Only
+  uses the auth section's password. (Default: Anonymous `tfa` authenticator)
+- `links`: How to represent external links. (Default: `fancy`)
+    - `ignore`: Don't download links.
+    - `plaintext`: A text file containing only the URL.
+    - `fancy`: A HTML file looking like the ILIAS link element.
+    - `internet-shortcut`: An internet shortcut file (`.url` file).
+- `link_redirect_delay`: Time (in seconds) until `fancy` link files will
+  redirect to the actual URL. Set to a negative value to disable the automatic
+  redirect. (Default: `-1`)
+- `videos`: Whether to download videos. (Default: `no`)
+- `forums`: Whether to download forum threads. (Default: `no`)
+- `http_timeout`: The timeout (in seconds) for all HTTP requests. (Default:
+  `20.0`)
+
+## Authenticator types
+
+### The `simple` authenticator
+
+With this authenticator, the username and password can be set directly in the
+config file. If the username or password are not specified, the user is prompted
+via the terminal.
+
+- `username`: The username. (Optional)
+- `password`: The password. (Optional)
+
+### The `credential-file` authenticator
+
+This authenticator reads a username and a password from a credential file.
+
+- `path`: Path to the credential file. (Required)
+
+The credential file has exactly two lines (trailing newline optional). The first
+line starts with `username=` and contains the username, the second line starts
+with `password=` and contains the password. The username and password may
+contain any characters except a line break.
+
+```
+username=AzureDiamond
+password=hunter2
+```
+
+### The `keyring` authenticator
+
+This authenticator uses the system keyring to store passwords. The username can
+be set directly in the config file. If the username is not specified, the user
+is prompted via the terminal. If the keyring contains no entry or the entry is
+incorrect, the user is prompted for a password via the terminal and the password
+is stored in the keyring.
+
+- `username`: The username. (Optional)
+- `keyring_name`: The service name PFERD uses for storing credentials. (Default:
+  `PFERD`)
+
+### The `pass` authenticator
+
+This authenticator queries the [`pass` password manager][3] for a username and
+password. It tries to be mostly compatible with [browserpass][4] and
+[passff][5], so see those links for an overview of the format. If PFERD fails
+to load your password, you can use the `--explain` flag to see why.
+
+- `passname`: The name of the password to use (Required)
+- `username_prefixes`: A comma-separated list of username line prefixes
+  (Default: `login,username,user`)
+- `password_prefixes`: A comma-separated list of password line prefixes
+  (Default: `password,pass,secret`)
+
+[3]: <https://www.passwordstore.org/> "Pass: The Standard Unix Password Manager"
+[4]: <https://github.com/browserpass/browserpass-extension#organizing-password-store> "Organizing password store"
+[5]: <https://github.com/passff/passff#multi-line-format> "Multi-line format"
+
+### The `tfa` authenticator
+
+This authenticator prompts the user on the console for a two-factor
+authentication token. The token is provided as password and it is not cached.
+This authenticator does not support usernames.
+
+## Transformation rules
+
+Transformation rules are rules for renaming and excluding files and directories.
+They are specified line-by-line in a crawler's `transform` option. When a
+crawler needs to apply a rule to a path, it goes through this list top-to-bottom
+and applies the first matching rule.
+
+To see this process in action, you can use the `--debug-transforms` or flag or
+the `--explain` flag.
+
+Each rule has the format `SOURCE ARROW TARGET` (e. g. `foo/bar --> foo/baz`).
+The arrow specifies how the source and target are interpreted. The different
+kinds of arrows are documented below.
+
+`SOURCE` and `TARGET` are either a bunch of characters without spaces (e. g.
+`foo/bar`) or string literals (e. g, `"foo/b a r"`). The former syntax has no
+concept of escaping characters, so the backslash is just another character. The
+string literals however support Python's escape syntax (e. g.
+`"foo\\bar\tbaz"`). This also means that in string literals, backslashes must be
+escaped.
+
+`TARGET` can additionally be a single exclamation mark `!` (*not* `"!"`). When a
+rule with a `!` as target matches a path, the corresponding file or directory is
+ignored by the crawler instead of renamed.
+
+`TARGET` can also be omitted entirely. When a rule without target matches a
+path, the path is returned unmodified. This is useful to prevent rules further
+down from matching instead.
+
+Each arrow's behaviour can be modified slightly by changing the arrow's head
+from `>` to `>>`. When a rule with a `>>` arrow head matches a path, it doesn't
+return immediately like a normal arrow. Instead, it replaces the current path
+with its output and continues on to the next rule. In effect, this means that
+multiple rules can be applied sequentially.
+
+### The `-->` arrow
+
+The `-->` arrow is a basic renaming operation for files and directories. If a
+path matches `SOURCE`, it is renamed to `TARGET`.
+
+Example: `foo/bar --> baz`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Converts `foo/bar` into `baz`
+- Converts `foo/bar/wargl` into `baz/wargl`
+
+Example: `foo/bar --> !`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Ignores `foo/bar` and any of its children
+
+### The `-name->` arrow
+
+The `-name->` arrow lets you rename files and directories by their name,
+regardless of where they appear in the file tree. Because of this, its `SOURCE`
+must not contain multiple path segments, only a single name. This restriction
+does not apply to its `TARGET`.
+
+Example: `foo -name-> bar/baz`
+- Doesn't match `a/foobar/b` or `x/Foo/y/z`
+- Converts `hello/foo` into `hello/bar/baz`
+- Converts `foo/world` into `bar/baz/world`
+- Converts `a/foo/b/c/foo` into `a/bar/baz/b/c/bar/baz`
+
+Example: `foo -name-> !`
+- Doesn't match `a/foobar/b` or `x/Foo/y/z`
+- Ignores any path containing a segment `foo`
+
+### The `-exact->` arrow
+
+The `-exact->` arrow requires the path to match `SOURCE` exactly. The examples
+below show why this is useful.
+
+Example: `foo/bar -exact-> baz`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Converts `foo/bar` into `baz`
+- Doesn't match `foo/bar/wargl`
+
+Example: `foo/bar -exact-> !`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Ignores only `foo/bar`, not its children
+
+### The `-re->` arrow
+
+The `-re->` arrow is like the `-->` arrow but with regular expressions. `SOURCE`
+is a regular expression and `TARGET` an f-string based template. If a path
+matches `SOURCE`, the output path is created using `TARGET` as template.
+`SOURCE` is automatically anchored.
+
+`TARGET` uses Python's [format string syntax][6]. The *n*-th capturing group can
+be referred to as `{g<n>}` (e.g. `{g3}`). `{g0}` refers to the original path.
+If capturing group *n*'s contents are a valid integer, the integer value is
+available as `{i<n>}` (e.g. `{i3}`). If capturing group *n*'s contents are a
+valid float, the float value is available as `{f<n>}` (e.g. `{f3}`). If a
+capturing group is not present (e.g. when matching the string `cd` with the
+regex `(ab)?cd`), the corresponding variables are not defined.
+
+Python's format string syntax has rich options for formatting its arguments. For
+example, to left-pad the capturing group 3 with the digit `0` to width 5, you
+can use `{i3:05}`.
+
+PFERD even allows you to write entire expressions inside the curly braces, for
+example `{g2.lower()}` or `{g3.replace(' ', '_')}`.
+
+Example: `f(oo+)/be?ar -re-> B{g1.upper()}H/fear`
+- Doesn't match `a/foo/bar`, `foo/abc/bar`, `afoo/bar` or `foo/bars`
+- Converts `foo/bar` into `BOOH/fear`
+- Converts `fooooo/bear` into `BOOOOOH/fear`
+- Converts `foo/bar/baz` into `BOOH/fear/baz`
+
+[6]: <https://docs.python.org/3/library/string.html#format-string-syntax> "Format String Syntax"
+
+### The `-name-re->` arrow
+
+The `-name-re>` arrow is like a combination of the `-name->` and `-re->` arrows.
+
+Example: `(.*)\.jpeg -name-re-> {g1}.jpg`
+- Doesn't match `foo/bar.png`, `baz.JPEG` or `hello,jpeg`
+- Converts `foo/bar.jpeg` into `foo/bar.jpg`
+- Converts `foo.jpeg/bar/baz.jpeg` into `foo.jpg/bar/baz.jpg`
+
+Example: `\..+ -name-re-> !`
+- Doesn't match `.`, `test`, `a.b`
+- Ignores all files and directories starting with `.`.
+
+### The `-exact-re->` arrow
+
+The `-exact-re>` arrow is like a combination of the `-exact->` and `-re->`
+arrows.
+
+Example: `f(oo+)/be?ar -exactre-> B{g1.upper()}H/fear`
+- Doesn't match `a/foo/bar`, `foo/abc/bar`, `afoo/bar` or `foo/bars`
+- Converts `foo/bar` into `BOOH/fear`
+- Converts `fooooo/bear` into `BOOOOOH/fear`
+- Doesn't match `foo/bar/baz`
+
+### Example: Tutorials
+
+You have an ILIAS course with lots of tutorials, but are only interested in a
+single one.
+
+```
+tutorials/
+  |- tut_01/
+  |- tut_02/
+  |- tut_03/
+  ...
+```
+
+You can use a mix of normal and exact arrows to get rid of the other ones and
+move the `tutorials/tut_02/` folder to `my_tut/`:
+
+```
+tutorials/tut_02 --> my_tut
+tutorials -exact->
+tutorials --> !
+```
+
+The second rule is required for many crawlers since they use the rules to decide
+which directories to crawl. If it was missing when the crawler looks at
+`tutorials/`, the third rule would match. This means the crawler would not crawl
+the `tutorials/` directory and thus not discover that `tutorials/tut02/` exists.
+
+Since the second rule is only relevant for crawling, the `TARGET` is left out.
+
+### Example: Lecture slides
+
+You have a course with slides like `Lecture 3: Linear functions.PDF` and you
+would like to rename them to `03_linear_functions.pdf`.
+
+```
+Lectures/
+  |- Lecture 1: Introduction.PDF
+  |- Lecture 2: Vectors and matrices.PDF
+  |- Lecture 3: Linear functions.PDF
+  ...
+```
+
+To do this, you can use the most powerful of arrows: The regex arrow.
+
+```
+"Lectures/Lecture (\\d+): (.*)\\.PDF" -re-> "Lectures/{i1:02}_{g2.lower().replace(' ', '_')}.pdf"
+```
+
+Note the escaped backslashes on the `SOURCE` side.
+
+### Example: Crawl a Python project
+
+You are crawling a Python project and want to ignore all hidden files (files
+whose name starts with a `.`), all `__pycache__` directories and all markdown
+files (for some weird reason).
+
+```
+.gitignore
+.mypy_cache/
+.venv/
+CONFIG.md
+PFERD/
+  |- __init__.py
+  |- __main__.py
+  |- __pycache__/
+  |- authenticator.py
+  |- config.py
+  ...
+README.md
+...
+```
+
+For this task, the name arrows can be used.
+
+```
+\..*        -name-re-> !
+__pycache__ -name->    !
+.*\.md      -name-re-> !
+```
+
+### Example: Clean up names
+
+You want to convert all paths into lowercase and replace spaces with underscores
+before applying any rules. This can be achieved using the `>>` arrow heads.
+
+```
+(.*) -re->> "{g1.lower().replace(' ', '_')}"
+
+<other rules go here>
+```

DEV.md

@@ -0,0 +1,89 @@
+# PFERD Development Guide
+
+PFERD is packaged following the [Python Packaging User Guide][ppug] (in
+particular [this][ppug-1] and [this][ppug-2] guide).
+
+[ppug]: <https://packaging.python.org/> "Python Packaging User Guide"
+[ppug-1]: <https://packaging.python.org/tutorials/packaging-projects/> "Packaging Python Projects"
+[ppug-2]: <https://packaging.python.org/guides/distributing-packages-using-setuptools/> "Packaging and distributing projects"
+
+## Setting up a dev environment
+
+The use of [venv][venv] is recommended. To initially set up a development
+environment, run these commands in the same directory as this file:
+
+```
+$ python -m venv .venv
+$ . .venv/bin/activate
+$ ./scripts/setup
+```
+
+The setup script installs a few required dependencies and tools. It also
+installs PFERD via `pip install --editable .`, which means that you can just run
+`pferd` as if it was installed normally. Since PFERD was installed with
+`--editable`, there is no need to re-run `pip install` when the source code is
+changed.
+
+If you get any errors because pip can't update itself, try running
+`./scripts/setup --no-pip` instead of `./scripts/setup`.
+
+For more details, see [this part of the Python Tutorial][venv-tut] and
+[this section on "development mode"][ppug-dev].
+
+[venv]: <https://docs.python.org/3/library/venv.html> "venv - Creation of virtual environments"
+[venv-tut]: <https://docs.python.org/3/tutorial/venv.html> "12. Virtual Environments and Packages"
+[ppug-dev]: <https://packaging.python.org/guides/distributing-packages-using-setuptools/#working-in-development-mode> "Working in “development mode”"
+
+## Checking and formatting the code
+
+To run a set of checks against the code, run `./scripts/check` in the repo's
+root directory. This script will run a few tools installed by `./scripts/setup`
+against the entire project.
+
+To format the code, run `./scripts/format` in the repo's root directory.
+
+Before committing changes, please make sure the checks return no warnings and
+the code is formatted.
+
+## Contributing
+
+When submitting a PR that adds, changes or modifies a feature, please ensure
+that the corresponding documentation is updated as well. Also, please ensure
+that `./scripts/check` returns no warnings and the code has been run through
+`./scripts/format`.
+
+In your first PR, please add your name to the `LICENSE` file.
+
+## Releasing a new version
+
+This section describes the steps required to release a new version of PFERD.
+Usually, they don't need to performed manually and `scripts/bump-version` can be
+used instead.
+
+1. Update the version number in `PFERD/version.py`
+2. Update `CHANGELOG.md`
+3. Commit changes to `master` with message `Bump version to <version>` (e. g. `Bump version to 3.2.5`)
+4. Create annotated tag named `v<version>` (e. g. `v3.2.5`)
+    - Copy changes from changelog
+    - Remove `#` symbols (which git would interpret as comments)
+    - As the first line, add `Version <version> - <date>` (e. g. `Version 3.2.5 - 2021-05-24`)
+    - Leave the second line empty
+5. Fast-forward `latest` to `master`
+6. Push `master`, `latest` and the new tag
+
+Example tag annotation:
+```
+Version 3.2.5 - 2021-05-24
+
+Added
+- Support for concurrent downloads
+- Support for proper config files
+- This changelog
+
+Changed
+- Rewrote almost everything
+- Redesigned CLI
+
+Removed
+- Backwards compatibility with 2.x
+```

LICENSE

@@ -1,4 +1,6 @@
-Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw
+Copyright 2019-2021 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
+                    TheChristophe, Scriptim, thelukasprobst, Toorero,
+                    Mr-Pine
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -15,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

PFERD/__init__.py

@@ -1,8 +0,0 @@
-# pylint: disable=invalid-name
-
-"""
-This module exports only what you need for a basic configuration. If you want a
-more complex configuration, you need to import the other submodules manually.
-"""
-
-from .pferd import Pferd

PFERD/__main__.py

@@ -0,0 +1,169 @@
+import argparse
+import asyncio
+import configparser
+import os
+import sys
+from pathlib import Path
+
+from .auth import AuthLoadError
+from .cli import PARSER, ParserLoadError, load_default_section
+from .config import Config, ConfigDumpError, ConfigLoadError, ConfigOptionError
+from .logging import log
+from .pferd import Pferd, PferdLoadError
+from .transformer import RuleParseError
+
+
+def load_config_parser(args: argparse.Namespace) -> configparser.ConfigParser:
+    log.explain_topic("Loading config")
+    parser = configparser.ConfigParser(interpolation=None)
+
+    if args.command is None:
+        log.explain("No CLI command specified, loading config from file")
+        Config.load_parser(parser, path=args.config)
+    else:
+        log.explain("CLI command specified, loading config from its arguments")
+        if args.command:
+            args.command(args, parser)
+
+    load_default_section(args, parser)
+
+    return parser
+
+
+def load_config(args: argparse.Namespace) -> Config:
+    try:
+        return Config(load_config_parser(args))
+    except ConfigLoadError as e:
+        log.error(str(e))
+        log.error_contd(e.reason)
+        sys.exit(1)
+    except ParserLoadError as e:
+        log.error(str(e))
+        sys.exit(1)
+
+
+def configure_logging_from_args(args: argparse.Namespace) -> None:
+    if args.explain is not None:
+        log.output_explain = args.explain
+    if args.status is not None:
+        log.output_status = args.status
+    if args.show_not_deleted is not None:
+        log.output_not_deleted = args.show_not_deleted
+    if args.report is not None:
+        log.output_report = args.report
+
+    # We want to prevent any unnecessary output if we're printing the config to
+    # stdout, otherwise it would not be a valid config file.
+    if args.dump_config_to == "-":
+        log.output_explain = False
+        log.output_status = False
+        log.output_report = False
+
+
+def configure_logging_from_config(args: argparse.Namespace, config: Config) -> None:
+    # In configure_logging_from_args(), all normal logging is already disabled
+    # whenever we dump the config. We don't want to override that decision with
+    # values from the config file.
+    if args.dump_config_to == "-":
+        return
+
+    try:
+        if args.explain is None:
+            log.output_explain = config.default_section.explain()
+        if args.status is None:
+            log.output_status = config.default_section.status()
+        if args.report is None:
+            log.output_report = config.default_section.report()
+        if args.show_not_deleted is None:
+            log.output_not_deleted = config.default_section.show_not_deleted()
+    except ConfigOptionError as e:
+        log.error(str(e))
+        sys.exit(1)
+
+
+def dump_config(args: argparse.Namespace, config: Config) -> None:
+    log.explain_topic("Dumping config")
+
+    if args.dump_config and args.dump_config_to is not None:
+        log.error("--dump-config and --dump-config-to can't be specified at the same time")
+        sys.exit(1)
+
+    try:
+        if args.dump_config:
+            config.dump()
+        elif args.dump_config_to == "-":
+            config.dump_to_stdout()
+        else:
+            config.dump(Path(args.dump_config_to))
+    except ConfigDumpError as e:
+        log.error(str(e))
+        log.error_contd(e.reason)
+        sys.exit(1)
+
+
+def main() -> None:
+    args = PARSER.parse_args()
+
+    # Configuring logging happens in two stages because CLI args have
+    # precedence over config file options and loading the config already
+    # produces some kinds of log messages (usually only explain()-s).
+    configure_logging_from_args(args)
+
+    config = load_config(args)
+
+    # Now, after loading the config file, we can apply its logging settings in
+    # all places that were not already covered by CLI args.
+    configure_logging_from_config(args, config)
+
+    if args.dump_config or args.dump_config_to is not None:
+        dump_config(args, config)
+        sys.exit()
+
+    try:
+        pferd = Pferd(config, args.crawler, args.skip)
+    except PferdLoadError as e:
+        log.unlock()
+        log.error(str(e))
+        sys.exit(1)
+
+    try:
+        if os.name == "nt":
+            # A "workaround" for the windows event loop somehow crashing after
+            # asyncio.run() completes. See:
+            # https://bugs.python.org/issue39232
+            # https://github.com/encode/httpx/issues/914#issuecomment-780023632
+            # TODO Fix this properly
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(pferd.run(args.debug_transforms))
+            loop.run_until_complete(asyncio.sleep(1))
+            loop.close()
+        else:
+            asyncio.run(pferd.run(args.debug_transforms))
+    except (ConfigOptionError, AuthLoadError) as e:
+        log.unlock()
+        log.error(str(e))
+        sys.exit(1)
+    except RuleParseError as e:
+        log.unlock()
+        e.pretty_print()
+        sys.exit(1)
+    except KeyboardInterrupt:
+        log.unlock()
+        log.explain_topic("Interrupted, exiting immediately")
+        log.explain("Open files and connections are left for the OS to clean up")
+        pferd.print_report()
+        # TODO Clean up tmp files
+        # And when those files *do* actually get cleaned up properly,
+        # reconsider if this should really exit with 1
+        sys.exit(1)
+    except Exception:
+        log.unlock()
+        log.unexpected_exception()
+        pferd.print_report()
+        sys.exit(1)
+    else:
+        pferd.print_report()
+
+
+if __name__ == "__main__":
+    main()

PFERD/auth/__init__.py

@@ -0,0 +1,29 @@
+from configparser import SectionProxy
+from typing import Callable, Dict
+
+from ..config import Config
+from .authenticator import Authenticator, AuthError, AuthLoadError, AuthSection  # noqa: F401
+from .credential_file import CredentialFileAuthenticator, CredentialFileAuthSection
+from .keyring import KeyringAuthenticator, KeyringAuthSection
+from .pass_ import PassAuthenticator, PassAuthSection
+from .simple import SimpleAuthenticator, SimpleAuthSection
+from .tfa import TfaAuthenticator
+
+AuthConstructor = Callable[[
+    str,                # Name (without the "auth:" prefix)
+    SectionProxy,       # Authenticator's section of global config
+    Config,             # Global config
+], Authenticator]
+
+AUTHENTICATORS: Dict[str, AuthConstructor] = {
+    "credential-file": lambda n, s, c:
+        CredentialFileAuthenticator(n, CredentialFileAuthSection(s), c),
+    "keyring": lambda n, s, c:
+        KeyringAuthenticator(n, KeyringAuthSection(s)),
+    "pass": lambda n, s, c:
+        PassAuthenticator(n, PassAuthSection(s)),
+    "simple": lambda n, s, c:
+        SimpleAuthenticator(n, SimpleAuthSection(s)),
+    "tfa": lambda n, s, c:
+        TfaAuthenticator(n),
+}

PFERD/auth/authenticator.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from ..config import Section
+
+
+class AuthLoadError(Exception):
+    pass
+
+
+class AuthError(Exception):
+    pass
+
+
+class AuthSection(Section):
+    def type(self) -> str:
+        value = self.s.get("type")
+        if value is None:
+            self.missing_value("type")
+        return value
+
+
+class Authenticator(ABC):
+    def __init__(self, name: str) -> None:
+        """
+        Initialize an authenticator from its name and its section in the config
+        file.
+
+        If you are writing your own constructor for your own authenticator,
+        make sure to call this constructor first (via super().__init__).
+
+        May throw an AuthLoadError.
+        """
+
+        self.name = name
+
+    @abstractmethod
+    async def credentials(self) -> Tuple[str, str]:
+        pass
+
+    async def username(self) -> str:
+        username, _ = await self.credentials()
+        return username
+
+    async def password(self) -> str:
+        _, password = await self.credentials()
+        return password
+
+    def invalidate_credentials(self) -> None:
+        """
+        Tell the authenticator that some or all of its credentials are invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid credentials")
+
+    def invalidate_username(self) -> None:
+        """
+        Tell the authenticator that specifically its username is invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid username")
+
+    def invalidate_password(self) -> None:
+        """
+        Tell the authenticator that specifically its password is invalid.
+
+        Authenticators should overwrite this function if they have a way to
+        deal with this issue that is likely to result in valid credentials
+        (e. g. prompting the user).
+        """
+
+        raise AuthError("Invalid password")

PFERD/auth/credential_file.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+from typing import Tuple
+
+from ..config import Config
+from ..utils import fmt_real_path
+from .authenticator import Authenticator, AuthLoadError, AuthSection
+
+
+class CredentialFileAuthSection(AuthSection):
+    def path(self) -> Path:
+        value = self.s.get("path")
+        if value is None:
+            self.missing_value("path")
+        return Path(value)
+
+
+class CredentialFileAuthenticator(Authenticator):
+    def __init__(self, name: str, section: CredentialFileAuthSection, config: Config) -> None:
+        super().__init__(name)
+
+        path = config.default_section.working_dir() / section.path()
+        try:
+            with open(path, encoding="utf-8") as f:
+                lines = list(f)
+        except UnicodeDecodeError:
+            raise AuthLoadError(f"Credential file at {fmt_real_path(path)} is not encoded using UTF-8")
+        except OSError as e:
+            raise AuthLoadError(f"No credential file at {fmt_real_path(path)}") from e
+
+        if len(lines) != 2:
+            raise AuthLoadError("Credential file must be two lines long")
+        [uline, pline] = lines
+        uline = uline[:-1]  # Remove trailing newline
+        if pline.endswith("\n"):
+            pline = pline[:-1]
+
+        if not uline.startswith("username="):
+            raise AuthLoadError("First line must start with 'username='")
+        if not pline.startswith("password="):
+            raise AuthLoadError("Second line must start with 'password='")
+
+        self._username = uline[9:]
+        self._password = pline[9:]
+
+    async def credentials(self) -> Tuple[str, str]:
+        return self._username, self._password

PFERD/auth/keyring.py

@@ -0,0 +1,65 @@
+from typing import Optional, Tuple
+
+import keyring
+
+from ..logging import log
+from ..utils import agetpass, ainput
+from ..version import NAME
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class KeyringAuthSection(AuthSection):
+    def username(self) -> Optional[str]:
+        return self.s.get("username")
+
+    def keyring_name(self) -> str:
+        return self.s.get("keyring_name", fallback=NAME)
+
+
+class KeyringAuthenticator(Authenticator):
+
+    def __init__(self, name: str, section: KeyringAuthSection) -> None:
+        super().__init__(name)
+
+        self._username = section.username()
+        self._password: Optional[str] = None
+        self._keyring_name = section.keyring_name()
+
+        self._password_invalidated = False
+        self._username_fixed = section.username() is not None
+
+    async def credentials(self) -> Tuple[str, str]:
+        # Request the username
+        if self._username is None:
+            async with log.exclusive_output():
+                self._username = await ainput("Username: ")
+
+        # First try looking it up in the keyring.
+        # Do not look it up if it was invalidated - we want to re-prompt in this case
+        if self._password is None and not self._password_invalidated:
+            self._password = keyring.get_password(self._keyring_name, self._username)
+
+        # If that fails it wasn't saved in the keyring - we need to
+        # read it from the user and store it
+        if self._password is None:
+            async with log.exclusive_output():
+                self._password = await agetpass("Password: ")
+                keyring.set_password(self._keyring_name, self._username, self._password)
+
+        self._password_invalidated = False
+        return self._username, self._password
+
+    def invalidate_credentials(self) -> None:
+        if not self._username_fixed:
+            self.invalidate_username()
+        self.invalidate_password()
+
+    def invalidate_username(self) -> None:
+        if self._username_fixed:
+            raise AuthError("Configured username is invalid")
+        else:
+            self._username = None
+
+    def invalidate_password(self) -> None:
+        self._password = None
+        self._password_invalidated = True

PFERD/auth/pass_.py

@@ -0,0 +1,98 @@
+import re
+import subprocess
+from typing import List, Tuple
+
+from ..logging import log
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class PassAuthSection(AuthSection):
+    def passname(self) -> str:
+        if (value := self.s.get("passname")) is None:
+            self.missing_value("passname")
+        return value
+
+    def username_prefixes(self) -> List[str]:
+        value = self.s.get("username_prefixes", "login,username,user")
+        return [prefix.lower() for prefix in value.split(",")]
+
+    def password_prefixes(self) -> List[str]:
+        value = self.s.get("password_prefixes", "password,pass,secret")
+        return [prefix.lower() for prefix in value.split(",")]
+
+
+class PassAuthenticator(Authenticator):
+    PREFIXED_LINE_RE = r"([a-zA-Z]+):\s?(.*)"  # to be used with fullmatch
+
+    def __init__(self, name: str, section: PassAuthSection) -> None:
+        super().__init__(name)
+
+        self._passname = section.passname()
+        self._username_prefixes = section.username_prefixes()
+        self._password_prefixes = section.password_prefixes()
+
+    async def credentials(self) -> Tuple[str, str]:
+        log.explain_topic("Obtaining credentials from pass")
+
+        try:
+            log.explain(f"Calling 'pass show {self._passname}'")
+            result = subprocess.check_output(["pass", "show", self._passname], text=True)
+        except subprocess.CalledProcessError as e:
+            raise AuthError(f"Failed to get password info from {self._passname}: {e}")
+
+        prefixed = {}
+        unprefixed = []
+        for line in result.strip().splitlines():
+            if match := re.fullmatch(self.PREFIXED_LINE_RE, line):
+                prefix = match.group(1).lower()
+                value = match.group(2)
+                log.explain(f"Found prefixed line {line!r} with prefix {prefix!r}, value {value!r}")
+                if prefix in prefixed:
+                    raise AuthError(f"Prefix {prefix} specified multiple times")
+                prefixed[prefix] = value
+            else:
+                log.explain(f"Found unprefixed line {line!r}")
+                unprefixed.append(line)
+
+        username = None
+        for prefix in self._username_prefixes:
+            log.explain(f"Looking for username at prefix {prefix!r}")
+            if prefix in prefixed:
+                username = prefixed[prefix]
+                log.explain(f"Found username {username!r}")
+                break
+
+        password = None
+        for prefix in self._password_prefixes:
+            log.explain(f"Looking for password at prefix {prefix!r}")
+            if prefix in prefixed:
+                password = prefixed[prefix]
+                log.explain(f"Found password {password!r}")
+                break
+
+        if password is None and username is None:
+            log.explain("No username and password found so far")
+            log.explain("Using first unprefixed line as password")
+            log.explain("Using second unprefixed line as username")
+        elif password is None:
+            log.explain("No password found so far")
+            log.explain("Using first unprefixed line as password")
+        elif username is None:
+            log.explain("No username found so far")
+            log.explain("Using first unprefixed line as username")
+
+        if password is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Password could not be determined")
+            password = unprefixed.pop(0)
+            log.explain(f"Found password {password!r}")
+
+        if username is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Username could not be determined")
+            username = unprefixed.pop(0)
+            log.explain(f"Found username {username!r}")
+
+        return username, password

PFERD/auth/simple.py

@@ -0,0 +1,62 @@
+from typing import Optional, Tuple
+
+from ..logging import log
+from ..utils import agetpass, ainput
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class SimpleAuthSection(AuthSection):
+    def username(self) -> Optional[str]:
+        return self.s.get("username")
+
+    def password(self) -> Optional[str]:
+        return self.s.get("password")
+
+
+class SimpleAuthenticator(Authenticator):
+    def __init__(self, name: str, section: SimpleAuthSection) -> None:
+        super().__init__(name)
+
+        self._username = section.username()
+        self._password = section.password()
+
+        self._username_fixed = self.username is not None
+        self._password_fixed = self.password is not None
+
+    async def credentials(self) -> Tuple[str, str]:
+        if self._username is not None and self._password is not None:
+            return self._username, self._password
+
+        async with log.exclusive_output():
+            if self._username is None:
+                self._username = await ainput("Username: ")
+            else:
+                print(f"Username: {self._username}")
+
+            if self._password is None:
+                self._password = await agetpass("Password: ")
+
+            # Intentionally returned inside the context manager so we know
+            # they're both not None
+            return self._username, self._password
+
+    def invalidate_credentials(self) -> None:
+        if self._username_fixed and self._password_fixed:
+            raise AuthError("Configured credentials are invalid")
+
+        if not self._username_fixed:
+            self._username = None
+        if not self._password_fixed:
+            self._password = None
+
+    def invalidate_username(self) -> None:
+        if self._username_fixed:
+            raise AuthError("Configured username is invalid")
+        else:
+            self._username = None
+
+    def invalidate_password(self) -> None:
+        if self._password_fixed:
+            raise AuthError("Configured password is invalid")
+        else:
+            self._password = None

PFERD/auth/tfa.py

@@ -0,0 +1,30 @@
+from typing import Tuple
+
+from ..logging import log
+from ..utils import ainput
+from .authenticator import Authenticator, AuthError
+
+
+class TfaAuthenticator(Authenticator):
+    def __init__(self, name: str) -> None:
+        super().__init__(name)
+
+    async def username(self) -> str:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    async def password(self) -> str:
+        async with log.exclusive_output():
+            code = await ainput("TFA code: ")
+            return code
+
+    async def credentials(self) -> Tuple[str, str]:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    def invalidate_username(self) -> None:
+        raise AuthError("TFA authenticator does not support usernames")
+
+    def invalidate_password(self) -> None:
+        pass
+
+    def invalidate_credentials(self) -> None:
+        pass

PFERD/authenticators.py

@@ -1,125 +0,0 @@
-"""
-General authenticators useful in many situations
-"""
-
-import getpass
-from typing import Optional, Tuple
-
-
-class TfaAuthenticator:
-    # pylint: disable=too-few-public-methods
-    """
-    An authenticator for a TFA token. Always prompts the user, as the token can not be cached.
-    """
-
-    def __init__(self, reason: str):
-        """
-        Create a new tfa authenticator.
-
-        Arguments:
-            reason {str} -- the reason for obtaining the credentials
-        """
-        self._reason = reason
-
-    def get_token(self) -> str:
-        # pylint: disable=no-self-use
-        """
-        Prompts the user for the token and returns it.
-        """
-        print(f"Enter credentials ({self._reason})")
-        return getpass.getpass("TFA Token: ")
-
-
-class UserPassAuthenticator:
-    """
-    An authenticator for username-password combinations that prompts the user
-    for missing information.
-    """
-
-    def __init__(
-            self,
-            reason: str,
-            username: Optional[str] = None,
-            password: Optional[str] = None,
-    ) -> None:
-        """
-        reason   - what the credentials are used for
-        username - the username (if already known)
-        password - the password (if already known)
-        """
-
-        self._reason = reason
-
-        self._given_username = username
-        self._given_password = password
-
-        self._username = username
-        self._password = password
-
-    def get_credentials(self) -> Tuple[str, str]:
-        """
-        Returns a tuple (username, password). Prompts user for username or
-        password when necessary.
-        """
-
-        if self._username is None and self._given_username is not None:
-            self._username = self._given_username
-
-        if self._password is None and self._given_password is not None:
-            self._password = self._given_password
-
-        if self._username is None or self._password is None:
-            print(f"Enter credentials ({self._reason})")
-
-        username: str
-        if self._username is None:
-            username = input("Username: ")
-            self._username = username
-        else:
-            username = self._username
-
-        password: str
-        if self._password is None:
-            password = getpass.getpass(prompt="Password: ")
-            self._password = password
-        else:
-            password = self._password
-
-        return (username, password)
-
-    @property
-    def username(self) -> str:
-        """
-        The username. Accessing this property may cause the authenticator to
-        prompt the user.
-        """
-
-        (username, _) = self.get_credentials()
-        return username
-
-    @property
-    def password(self) -> str:
-        """
-        The password. Accessing this property may cause the authenticator to
-        prompt the user.
-        """
-
-        (_, password) = self.get_credentials()
-        return password
-
-    def invalidate_credentials(self) -> None:
-        """
-        Marks the credentials as invalid. If only a username was supplied in
-        the constructor, assumes that the username is valid and only the
-        password is invalid. If only a password was supplied in the
-        constructor, assumes that the password is valid and only the username
-        is invalid. Otherwise, assumes that username and password are both
-        invalid.
-        """
-
-        self._username = None
-        self._password = None
-
-        if self._given_username is not None and self._given_password is not None:
-            self._given_username = None
-            self._given_password = None

PFERD/cli/__init__.py

@@ -0,0 +1,13 @@
+# isort: skip_file
+
+# The order of imports matters because each command module registers itself
+# with the parser from ".parser" and the import order affects the order in
+# which they appear in the help. Because of this, isort is disabled for this
+# file. Also, since we're reexporting or just using the side effect of
+# importing itself, we get a few linting warnings, which we're disabling as
+# well.
+
+from . import command_local  # noqa: F401 imported but unused
+from . import command_kit_ilias_web  # noqa: F401 imported but unused
+from . import command_kit_ipd  # noqa: F401 imported but unused
+from .parser import PARSER, ParserLoadError, load_default_section  # noqa: F401 imported but unused

PFERD/cli/command_kit_ilias_web.py

@@ -0,0 +1,120 @@
+import argparse
+import configparser
+from pathlib import Path
+
+from ..crawl.ilias.file_templates import Links
+from ..logging import log
+from .parser import (CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, ParserLoadError, load_crawler,
+                     show_value_error)
+
+SUBPARSER = SUBPARSERS.add_parser(
+    "kit-ilias-web",
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title="kit-ilias-web crawler arguments",
+    description="arguments for the 'kit-ilias-web' crawler",
+)
+GROUP.add_argument(
+    "target",
+    type=str,
+    metavar="TARGET",
+    help="course id, 'desktop', or ILIAS URL to crawl"
+)
+GROUP.add_argument(
+    "output",
+    type=Path,
+    metavar="OUTPUT",
+    help="output directory"
+)
+GROUP.add_argument(
+    "--username", "-u",
+    type=str,
+    metavar="USERNAME",
+    help="user name for authentication"
+)
+GROUP.add_argument(
+    "--keyring",
+    action=BooleanOptionalAction,
+    help="use the system keyring to store and retrieve passwords"
+)
+GROUP.add_argument(
+    "--credential-file",
+    type=Path,
+    metavar="PATH",
+    help="read username and password from a credential file"
+)
+GROUP.add_argument(
+    "--links",
+    type=show_value_error(Links.from_string),
+    metavar="OPTION",
+    help="how to represent external links"
+)
+GROUP.add_argument(
+    "--link-redirect-delay",
+    type=int,
+    metavar="SECONDS",
+    help="time before 'fancy' links redirect to to their target (-1 to disable)"
+)
+GROUP.add_argument(
+    "--videos",
+    action=BooleanOptionalAction,
+    help="crawl and download videos"
+)
+GROUP.add_argument(
+    "--forums",
+    action=BooleanOptionalAction,
+    help="crawl and download forum posts"
+)
+GROUP.add_argument(
+    "--http-timeout", "-t",
+    type=float,
+    metavar="SECONDS",
+    help="timeout for all HTTP requests"
+)
+
+
+def load(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    log.explain("Creating config for command 'kit-ilias-web'")
+
+    parser["crawl:ilias"] = {}
+    section = parser["crawl:ilias"]
+    load_crawler(args, section)
+
+    section["type"] = "kit-ilias-web"
+    section["target"] = str(args.target)
+    section["output_dir"] = str(args.output)
+    section["auth"] = "auth:ilias"
+    if args.links is not None:
+        section["links"] = str(args.links.value)
+    if args.link_redirect_delay is not None:
+        section["link_redirect_delay"] = str(args.link_redirect_delay)
+    if args.videos is not None:
+        section["videos"] = "yes" if args.videos else "no"
+    if args.forums is not None:
+        section["forums"] = "yes" if args.forums else "no"
+    if args.http_timeout is not None:
+        section["http_timeout"] = str(args.http_timeout)
+
+    parser["auth:ilias"] = {}
+    auth_section = parser["auth:ilias"]
+    if args.credential_file is not None:
+        if args.username is not None:
+            raise ParserLoadError("--credential-file and --username can't be used together")
+        if args.keyring:
+            raise ParserLoadError("--credential-file and --keyring can't be used together")
+        auth_section["type"] = "credential-file"
+        auth_section["path"] = str(args.credential_file)
+    elif args.keyring:
+        auth_section["type"] = "keyring"
+    else:
+        auth_section["type"] = "simple"
+    if args.username is not None:
+        auth_section["username"] = args.username
+
+
+SUBPARSER.set_defaults(command=load)

PFERD/cli/command_kit_ipd.py

@@ -0,0 +1,54 @@
+import argparse
+import configparser
+from pathlib import Path
+
+from ..logging import log
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
+
+SUBPARSER = SUBPARSERS.add_parser(
+    "kit-ipd",
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title="kit ipd crawler arguments",
+    description="arguments for the 'kit-ipd' crawler",
+)
+GROUP.add_argument(
+    "--link-regex",
+    type=str,
+    metavar="REGEX",
+    help="href-matching regex to identify downloadable files"
+)
+GROUP.add_argument(
+    "target",
+    type=str,
+    metavar="TARGET",
+    help="url to crawl"
+)
+GROUP.add_argument(
+    "output",
+    type=Path,
+    metavar="OUTPUT",
+    help="output directory"
+)
+
+
+def load(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    log.explain("Creating config for command 'kit-ipd'")
+
+    parser["crawl:kit-ipd"] = {}
+    section = parser["crawl:kit-ipd"]
+    load_crawler(args, section)
+
+    section["type"] = "kit-ipd"
+    section["target"] = str(args.target)
+    section["output_dir"] = str(args.output)
+    if args.link_regex:
+        section["link_regex"] = str(args.link_regex)
+
+
+SUBPARSER.set_defaults(command=load)

PFERD/cli/command_local.py

@@ -0,0 +1,70 @@
+import argparse
+import configparser
+from pathlib import Path
+
+from ..logging import log
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
+
+SUBPARSER = SUBPARSERS.add_parser(
+    "local",
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title="local crawler arguments",
+    description="arguments for the 'local' crawler",
+)
+GROUP.add_argument(
+    "target",
+    type=Path,
+    metavar="TARGET",
+    help="directory to crawl"
+)
+GROUP.add_argument(
+    "output",
+    type=Path,
+    metavar="OUTPUT",
+    help="output directory"
+)
+GROUP.add_argument(
+    "--crawl-delay",
+    type=float,
+    metavar="SECONDS",
+    help="artificial delay to simulate for crawl requests"
+)
+GROUP.add_argument(
+    "--download-delay",
+    type=float,
+    metavar="SECONDS",
+    help="artificial delay to simulate for download requests"
+)
+GROUP.add_argument(
+    "--download-speed",
+    type=int,
+    metavar="BYTES_PER_SECOND",
+    help="download speed to simulate"
+)
+
+
+def load(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    log.explain("Creating config for command 'local'")
+
+    parser["crawl:local"] = {}
+    section = parser["crawl:local"]
+    load_crawler(args, section)
+
+    section["type"] = "local"
+    section["target"] = str(args.target)
+    section["output_dir"] = str(args.output)
+    if args.crawl_delay is not None:
+        section["crawl_delay"] = str(args.crawl_delay)
+    if args.download_delay is not None:
+        section["download_delay"] = str(args.download_delay)
+    if args.download_speed is not None:
+        section["download_speed"] = str(args.download_speed)
+
+
+SUBPARSER.set_defaults(command=load)

PFERD/cli/parser.py

@@ -0,0 +1,245 @@
+import argparse
+import configparser
+from argparse import ArgumentTypeError
+from pathlib import Path
+from typing import Any, Callable, List, Optional, Sequence, Union
+
+from ..output_dir import OnConflict, Redownload
+from ..version import NAME, VERSION
+
+
+class ParserLoadError(Exception):
+    pass
+
+
+# TODO Replace with argparse version when updating to 3.9?
+class BooleanOptionalAction(argparse.Action):
+    def __init__(
+            self,
+            option_strings: List[str],
+            dest: Any,
+            default: Any = None,
+            type: Any = None,
+            choices: Any = None,
+            required: Any = False,
+            help: Any = None,
+            metavar: Any = None,
+    ):
+        if len(option_strings) != 1:
+            raise ValueError("There must be exactly one option string")
+        [self.name] = option_strings
+        if not self.name.startswith("--"):
+            raise ValueError(f"{self.name!r} doesn't start with '--'")
+        if self.name.startswith("--no-"):
+            raise ValueError(f"{self.name!r} starts with '--no-'")
+
+        options = [self.name, "--no-" + self.name[2:]]
+
+        super().__init__(
+            options,
+            dest,
+            nargs=0,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar,
+        )
+
+    def __call__(
+            self,
+            parser: argparse.ArgumentParser,
+            namespace: argparse.Namespace,
+            values: Union[str, Sequence[Any], None],
+            option_string: Optional[str] = None,
+    ) -> None:
+        if option_string and option_string in self.option_strings:
+            value = not option_string.startswith("--no-")
+            setattr(namespace, self.dest, value)
+
+    def format_usage(self) -> str:
+        return "--[no-]" + self.name[2:]
+
+
+def show_value_error(inner: Callable[[str], Any]) -> Callable[[str], Any]:
+    """
+    Some validation functions (like the from_string in our enums) raise a ValueError.
+    Argparse only pretty-prints ArgumentTypeErrors though, so we need to wrap our ValueErrors.
+    """
+    def wrapper(input: str) -> Any:
+        try:
+            return inner(input)
+        except ValueError as e:
+            raise ArgumentTypeError(e)
+    return wrapper
+
+
+CRAWLER_PARSER = argparse.ArgumentParser(add_help=False)
+CRAWLER_PARSER_GROUP = CRAWLER_PARSER.add_argument_group(
+    title="general crawler arguments",
+    description="arguments common to all crawlers",
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--redownload", "-r",
+    type=show_value_error(Redownload.from_string),
+    metavar="OPTION",
+    help="when to download a file that's already present locally"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--on-conflict",
+    type=show_value_error(OnConflict.from_string),
+    metavar="OPTION",
+    help="what to do when local and remote files or directories differ"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--transform", "-T",
+    action="append",
+    type=str,
+    metavar="RULE",
+    help="add a single transformation rule. Can be specified multiple times"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--tasks", "-n",
+    type=int,
+    metavar="N",
+    help="maximum number of concurrent tasks (crawling, downloading)"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--downloads", "-N",
+    type=int,
+    metavar="N",
+    help="maximum number of tasks that may download data at the same time"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--task-delay", "-d",
+    type=float,
+    metavar="SECONDS",
+    help="time the crawler should wait between subsequent tasks"
+)
+CRAWLER_PARSER_GROUP.add_argument(
+    "--windows-paths",
+    action=BooleanOptionalAction,
+    help="whether to repair invalid paths on windows"
+)
+
+
+def load_crawler(
+        args: argparse.Namespace,
+        section: configparser.SectionProxy,
+) -> None:
+    if args.redownload is not None:
+        section["redownload"] = args.redownload.value
+    if args.on_conflict is not None:
+        section["on_conflict"] = args.on_conflict.value
+    if args.transform is not None:
+        section["transform"] = "\n" + "\n".join(args.transform)
+    if args.tasks is not None:
+        section["tasks"] = str(args.tasks)
+    if args.downloads is not None:
+        section["downloads"] = str(args.downloads)
+    if args.task_delay is not None:
+        section["task_delay"] = str(args.task_delay)
+    if args.windows_paths is not None:
+        section["windows_paths"] = "yes" if args.windows_paths else "no"
+
+
+PARSER = argparse.ArgumentParser()
+PARSER.set_defaults(command=None)
+PARSER.add_argument(
+    "--version",
+    action="version",
+    version=f"{NAME} {VERSION} (https://github.com/Garmelon/PFERD)",
+)
+PARSER.add_argument(
+    "--config", "-c",
+    type=Path,
+    metavar="PATH",
+    help="custom config file"
+)
+PARSER.add_argument(
+    "--dump-config",
+    action="store_true",
+    help="dump current configuration to the default config path and exit"
+)
+PARSER.add_argument(
+    "--dump-config-to",
+    metavar="PATH",
+    help="dump current configuration to a file and exit."
+    " Use '-' as path to print to stdout instead"
+)
+PARSER.add_argument(
+    "--debug-transforms",
+    action="store_true",
+    help="apply transform rules to files of previous run"
+)
+PARSER.add_argument(
+    "--crawler", "-C",
+    action="append",
+    type=str,
+    metavar="NAME",
+    help="only execute a single crawler."
+    " Can be specified multiple times to execute multiple crawlers"
+)
+PARSER.add_argument(
+    "--skip", "-S",
+    action="append",
+    type=str,
+    metavar="NAME",
+    help="don't execute this particular crawler."
+    " Can be specified multiple times to skip multiple crawlers"
+)
+PARSER.add_argument(
+    "--working-dir",
+    type=Path,
+    metavar="PATH",
+    help="custom working directory"
+)
+PARSER.add_argument(
+    "--explain",
+    action=BooleanOptionalAction,
+    help="log and explain in detail what PFERD is doing"
+)
+PARSER.add_argument(
+    "--status",
+    action=BooleanOptionalAction,
+    help="print status updates while PFERD is crawling"
+)
+PARSER.add_argument(
+    "--report",
+    action=BooleanOptionalAction,
+    help="print a report of all local changes before exiting"
+)
+PARSER.add_argument(
+    "--share-cookies",
+    action=BooleanOptionalAction,
+    help="whether crawlers should share cookies where applicable"
+)
+PARSER.add_argument(
+    "--show-not-deleted",
+    action=BooleanOptionalAction,
+    help="print messages in status and report when PFERD did not delete a local only file"
+)
+
+
+def load_default_section(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    section = parser[parser.default_section]
+
+    if args.working_dir is not None:
+        section["working_dir"] = str(args.working_dir)
+    if args.explain is not None:
+        section["explain"] = "yes" if args.explain else "no"
+    if args.status is not None:
+        section["status"] = "yes" if args.status else "no"
+    if args.report is not None:
+        section["report"] = "yes" if args.report else "no"
+    if args.share_cookies is not None:
+        section["share_cookies"] = "yes" if args.share_cookies else "no"
+    if args.show_not_deleted is not None:
+        section["show_not_deleted"] = "yes" if args.show_not_deleted else "no"
+
+
+SUBPARSERS = PARSER.add_subparsers(title="crawlers")

PFERD/config.py

@@ -0,0 +1,193 @@
+import asyncio
+import os
+import sys
+from configparser import ConfigParser, SectionProxy
+from pathlib import Path
+from typing import Any, List, NoReturn, Optional, Tuple
+
+from rich.markup import escape
+
+from .logging import log
+from .utils import fmt_real_path, prompt_yes_no
+
+
+class ConfigLoadError(Exception):
+    """
+    Something went wrong while loading the config from a file.
+    """
+
+    def __init__(self, path: Path, reason: str):
+        super().__init__(f"Failed to load config from {fmt_real_path(path)}")
+        self.path = path
+        self.reason = reason
+
+
+class ConfigOptionError(Exception):
+    """
+    An option in the config file has an invalid or missing value.
+    """
+
+    def __init__(self, section: str, key: str, desc: str):
+        super().__init__(f"Section {section!r}, key {key!r}: {desc}")
+        self.section = section
+        self.key = key
+        self.desc = desc
+
+
+class ConfigDumpError(Exception):
+    def __init__(self, path: Path, reason: str):
+        super().__init__(f"Failed to dump config to {fmt_real_path(path)}")
+        self.path = path
+        self.reason = reason
+
+
+class Section:
+    """
+    Base class for the crawler and auth section classes.
+    """
+
+    def __init__(self, section: SectionProxy):
+        self.s = section
+
+    def error(self, key: str, desc: str) -> NoReturn:
+        raise ConfigOptionError(self.s.name, key, desc)
+
+    def invalid_value(
+            self,
+            key: str,
+            value: Any,
+            reason: Optional[str],
+    ) -> NoReturn:
+        if reason is None:
+            self.error(key, f"Invalid value {value!r}")
+        else:
+            self.error(key, f"Invalid value {value!r}: {reason}")
+
+    def missing_value(self, key: str) -> NoReturn:
+        self.error(key, "Missing value")
+
+
+class DefaultSection(Section):
+    def working_dir(self) -> Path:
+        # TODO Change to working dir instead of manually prepending it to paths
+        pathstr = self.s.get("working_dir", ".")
+        return Path(pathstr).expanduser()
+
+    def explain(self) -> bool:
+        return self.s.getboolean("explain", fallback=False)
+
+    def status(self) -> bool:
+        return self.s.getboolean("status", fallback=True)
+
+    def report(self) -> bool:
+        return self.s.getboolean("report", fallback=True)
+
+    def show_not_deleted(self) -> bool:
+        return self.s.getboolean("show_not_deleted", fallback=True)
+
+    def share_cookies(self) -> bool:
+        return self.s.getboolean("share_cookies", fallback=True)
+
+
+class Config:
+    @staticmethod
+    def _default_path() -> Path:
+        if os.name == "posix":
+            return Path("~/.config/PFERD/pferd.cfg").expanduser()
+        elif os.name == "nt":
+            return Path("~/AppData/Roaming/PFERD/pferd.cfg").expanduser()
+        else:
+            return Path("~/.pferd.cfg").expanduser()
+
+    def __init__(self, parser: ConfigParser):
+        self._parser = parser
+        self._default_section = DefaultSection(parser[parser.default_section])
+
+    @property
+    def default_section(self) -> DefaultSection:
+        return self._default_section
+
+    @staticmethod
+    def load_parser(parser: ConfigParser, path: Optional[Path] = None) -> None:
+        """
+        May throw a ConfigLoadError.
+        """
+
+        if path:
+            log.explain("Path specified on CLI")
+        else:
+            log.explain("Using default path")
+            path = Config._default_path()
+        log.explain(f"Loading {fmt_real_path(path)}")
+
+        # Using config.read_file instead of config.read because config.read
+        # would just ignore a missing file and carry on.
+        try:
+            with open(path, encoding="utf-8") as f:
+                parser.read_file(f, source=str(path))
+        except FileNotFoundError:
+            raise ConfigLoadError(path, "File does not exist")
+        except IsADirectoryError:
+            raise ConfigLoadError(path, "That's a directory, not a file")
+        except PermissionError:
+            raise ConfigLoadError(path, "Insufficient permissions")
+        except UnicodeDecodeError:
+            raise ConfigLoadError(path, "File is not encoded using UTF-8")
+
+    def dump(self, path: Optional[Path] = None) -> None:
+        """
+        May throw a ConfigDumpError.
+        """
+
+        if path:
+            log.explain("Using custom path")
+        else:
+            log.explain("Using default path")
+            path = self._default_path()
+
+        log.explain(f"Dumping to {fmt_real_path(path)}")
+        log.print(f"[bold bright_cyan]Dumping[/] to {escape(fmt_real_path(path))}")
+
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+        except PermissionError:
+            raise ConfigDumpError(path, "Could not create parent directory")
+
+        try:
+            # Ensuring we don't accidentally overwrite any existing files by
+            # always asking before overwriting a file.
+            try:
+                # x = open for exclusive creation, failing if the file already
+                # exists
+                with open(path, "x", encoding="utf-8") as f:
+                    self._parser.write(f)
+            except FileExistsError:
+                print("That file already exists.")
+                if asyncio.run(prompt_yes_no("Overwrite it?", default=False)):
+                    with open(path, "w", encoding="utf-8") as f:
+                        self._parser.write(f)
+                else:
+                    raise ConfigDumpError(path, "File already exists")
+        except IsADirectoryError:
+            raise ConfigDumpError(path, "That's a directory, not a file")
+        except PermissionError:
+            raise ConfigDumpError(path, "Insufficient permissions")
+
+    def dump_to_stdout(self) -> None:
+        self._parser.write(sys.stdout)
+
+    def crawl_sections(self) -> List[Tuple[str, SectionProxy]]:
+        result = []
+        for name, proxy in self._parser.items():
+            if name.startswith("crawl:"):
+                result.append((name, proxy))
+
+        return result
+
+    def auth_sections(self) -> List[Tuple[str, SectionProxy]]:
+        result = []
+        for name, proxy in self._parser.items():
+            if name.startswith("auth:"):
+                result.append((name, proxy))
+
+        return result

PFERD/cookie_jar.py

@@ -1,69 +0,0 @@
-"""A helper for requests cookies."""
-
-import logging
-from http.cookiejar import LoadError, LWPCookieJar
-from pathlib import Path
-from typing import Optional
-
-import requests
-
-LOGGER = logging.getLogger(__name__)
-
-
-class CookieJar:
-    """A cookie jar that can be persisted."""
-
-    def __init__(self, cookie_file: Optional[Path] = None) -> None:
-        """Create a new cookie jar at the given path.
-
-        If the path is None, the cookies will not be persisted.
-        """
-        self._cookies: LWPCookieJar
-        if cookie_file is None:
-            self._cookies = LWPCookieJar()
-        else:
-            self._cookies = LWPCookieJar(str(cookie_file.resolve()))
-
-    @property
-    def cookies(self) -> LWPCookieJar:
-        """Return the requests cookie jar."""
-        return self._cookies
-
-    def load_cookies(self) -> None:
-        """Load all cookies from the file given in the constructor."""
-        if self._cookies.filename is None:
-            return
-
-        try:
-            LOGGER.info("Loading old cookies from %s", self._cookies.filename)
-            self._cookies.load(ignore_discard=True)
-        except (FileNotFoundError, LoadError):
-            LOGGER.warning(
-                "No valid cookie file found at %s, continuing with no cookies",
-                self._cookies.filename
-            )
-
-    def save_cookies(self, reason: Optional[str] = None) -> None:
-        """Save the cookies in the file given in the constructor."""
-        if self._cookies.filename is None:
-            return
-
-        if reason is None:
-            LOGGER.info("Saving cookies")
-        else:
-            LOGGER.info("Saving cookies (%s)", reason)
-
-        # TODO figure out why ignore_discard is set
-        # TODO possibly catch a few more exceptions
-        self._cookies.save(ignore_discard=True)
-
-    def create_session(self) -> requests.Session:
-        """Create a new session using the cookie jar."""
-        sess = requests.Session()
-
-        # From the request docs: "All requests code should work out of the box
-        # with externally provided instances of CookieJar, e.g. LWPCookieJar
-        # and FileCookieJar."
-        sess.cookies = self.cookies  # type: ignore
-
-        return sess

PFERD/crawl/__init__.py

@@ -0,0 +1,25 @@
+from configparser import SectionProxy
+from typing import Callable, Dict
+
+from ..auth import Authenticator
+from ..config import Config
+from .crawler import Crawler, CrawlError, CrawlerSection  # noqa: F401
+from .ilias import KitIliasWebCrawler, KitIliasWebCrawlerSection
+from .kit_ipd_crawler import KitIpdCrawler, KitIpdCrawlerSection
+from .local_crawler import LocalCrawler, LocalCrawlerSection
+
+CrawlerConstructor = Callable[[
+    str,                       # Name (without the "crawl:" prefix)
+    SectionProxy,              # Crawler's section of global config
+    Config,                    # Global config
+    Dict[str, Authenticator],  # Loaded authenticators by name
+], Crawler]
+
+CRAWLERS: Dict[str, CrawlerConstructor] = {
+    "local": lambda n, s, c, a:
+        LocalCrawler(n, LocalCrawlerSection(s), c),
+    "kit-ilias-web": lambda n, s, c, a:
+        KitIliasWebCrawler(n, KitIliasWebCrawlerSection(s), c, a),
+    "kit-ipd": lambda n, s, c, a:
+        KitIpdCrawler(n, KitIpdCrawlerSection(s), c),
+}

PFERD/crawl/crawler.py

@@ -0,0 +1,369 @@
+import asyncio
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Awaitable, Coroutine
+from datetime import datetime
+from pathlib import Path, PurePath
+from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, TypeVar
+
+from ..auth import Authenticator
+from ..config import Config, Section
+from ..deduplicator import Deduplicator
+from ..limiter import Limiter
+from ..logging import ProgressBar, log
+from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload
+from ..report import MarkConflictError, MarkDuplicateError, Report
+from ..transformer import Transformer
+from ..utils import ReusableAsyncContextManager, fmt_path
+
+
+class CrawlWarning(Exception):
+    pass
+
+
+class CrawlError(Exception):
+    pass
+
+
+Wrapped = TypeVar("Wrapped", bound=Callable[..., None])
+
+
+def noncritical(f: Wrapped) -> Wrapped:
+    """
+    Catches and logs a few noncritical exceptions occurring during the function
+    call, mainly CrawlWarning.
+
+    If any exception occurs during the function call, the crawler's error_free
+    variable is set to False. This includes noncritical exceptions.
+
+    Warning: Must only be applied to member functions of the Crawler class!
+    """
+
+    def wrapper(*args: Any, **kwargs: Any) -> None:
+        if not (args and isinstance(args[0], Crawler)):
+            raise RuntimeError("@noncritical must only applied to Crawler methods")
+
+        crawler = args[0]
+
+        try:
+            f(*args, **kwargs)
+        except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
+            crawler.report.add_warning(str(e))
+            log.warn(str(e))
+            crawler.error_free = False
+        except Exception as e:
+            crawler.error_free = False
+            crawler.report.add_error(str(e))
+            raise
+
+    return wrapper  # type: ignore
+
+
+AWrapped = TypeVar("AWrapped", bound=Callable[..., Coroutine[Any, Any, Optional[Any]]])
+
+
+def anoncritical(f: AWrapped) -> AWrapped:
+    """
+    An async version of @noncritical.
+
+    Catches and logs a few noncritical exceptions occurring during the function
+    call, mainly CrawlWarning.
+
+    If any exception occurs during the function call, the crawler's error_free
+    variable is set to False. This includes noncritical exceptions.
+
+    Warning: Must only be applied to member functions of the Crawler class!
+    """
+
+    async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
+        if not (args and isinstance(args[0], Crawler)):
+            raise RuntimeError("@anoncritical must only applied to Crawler methods")
+
+        crawler = args[0]
+
+        try:
+            return await f(*args, **kwargs)
+        except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
+            log.warn(str(e))
+            crawler.error_free = False
+            crawler.report.add_warning(str(e))
+        except Exception as e:
+            crawler.error_free = False
+            crawler.report.add_error(str(e))
+            raise
+
+        return None
+
+    return wrapper  # type: ignore
+
+
+class CrawlToken(ReusableAsyncContextManager[ProgressBar]):
+    def __init__(self, limiter: Limiter, path: PurePath):
+        super().__init__()
+
+        self._limiter = limiter
+        self._path = path
+
+    @property
+    def path(self) -> PurePath:
+        return self._path
+
+    async def _on_aenter(self) -> ProgressBar:
+        self._stack.callback(lambda: log.status("[bold cyan]", "Crawled", fmt_path(self._path)))
+        await self._stack.enter_async_context(self._limiter.limit_crawl())
+        bar = self._stack.enter_context(log.crawl_bar("[bold bright_cyan]", "Crawling", fmt_path(self._path)))
+
+        return bar
+
+
+class DownloadToken(ReusableAsyncContextManager[Tuple[ProgressBar, FileSink]]):
+    def __init__(self, limiter: Limiter, fs_token: FileSinkToken, path: PurePath):
+        super().__init__()
+
+        self._limiter = limiter
+        self._fs_token = fs_token
+        self._path = path
+
+    @property
+    def path(self) -> PurePath:
+        return self._path
+
+    async def _on_aenter(self) -> Tuple[ProgressBar, FileSink]:
+        await self._stack.enter_async_context(self._limiter.limit_download())
+        sink = await self._stack.enter_async_context(self._fs_token)
+        # The "Downloaded ..." message is printed in the output dir, not here
+        bar = self._stack.enter_context(log.download_bar("[bold bright_cyan]", "Downloading",
+                                                         fmt_path(self._path)))
+
+        return bar, sink
+
+
+class CrawlerSection(Section):
+    def type(self) -> str:
+        value = self.s.get("type")
+        if value is None:
+            self.missing_value("type")
+        return value
+
+    def skip(self) -> bool:
+        return self.s.getboolean("skip", fallback=False)
+
+    def output_dir(self, name: str) -> Path:
+        # TODO Use removeprefix() after switching to 3.9
+        if name.startswith("crawl:"):
+            name = name[len("crawl:"):]
+        return Path(self.s.get("output_dir", name)).expanduser()
+
+    def redownload(self) -> Redownload:
+        value = self.s.get("redownload", "never-smart")
+        try:
+            return Redownload.from_string(value)
+        except ValueError as e:
+            self.invalid_value(
+                "redownload",
+                value,
+                str(e).capitalize(),
+            )
+
+    def on_conflict(self) -> OnConflict:
+        value = self.s.get("on_conflict", "prompt")
+        try:
+            return OnConflict.from_string(value)
+        except ValueError as e:
+            self.invalid_value(
+                "on_conflict",
+                value,
+                str(e).capitalize(),
+            )
+
+    def transform(self) -> str:
+        return self.s.get("transform", "")
+
+    def tasks(self) -> int:
+        value = self.s.getint("tasks", fallback=1)
+        if value <= 0:
+            self.invalid_value("tasks", value, "Must be greater than 0")
+        return value
+
+    def downloads(self) -> int:
+        tasks = self.tasks()
+        value = self.s.getint("downloads", fallback=None)
+        if value is None:
+            return tasks
+        if value <= 0:
+            self.invalid_value("downloads", value, "Must be greater than 0")
+        if value > tasks:
+            self.invalid_value("downloads", value, "Must not be greater than tasks")
+        return value
+
+    def task_delay(self) -> float:
+        value = self.s.getfloat("task_delay", fallback=0.0)
+        if value < 0:
+            self.invalid_value("task_delay", value, "Must not be negative")
+        return value
+
+    def windows_paths(self) -> bool:
+        on_windows = os.name == "nt"
+        return self.s.getboolean("windows_paths", fallback=on_windows)
+
+    def auth(self, authenticators: Dict[str, Authenticator]) -> Authenticator:
+        value = self.s.get("auth")
+        if value is None:
+            self.missing_value("auth")
+        auth = authenticators.get(value)
+        if auth is None:
+            self.invalid_value("auth", value, "No such auth section exists")
+        return auth
+
+
+class Crawler(ABC):
+    def __init__(
+            self,
+            name: str,
+            section: CrawlerSection,
+            config: Config,
+    ) -> None:
+        """
+        Initialize a crawler from its name and its section in the config file.
+
+        If you are writing your own constructor for your own crawler, make sure
+        to call this constructor first (via super().__init__).
+
+        May throw a CrawlerLoadException.
+        """
+
+        self.name = name
+        self.error_free = True
+
+        self._limiter = Limiter(
+            task_limit=section.tasks(),
+            download_limit=section.downloads(),
+            task_delay=section.task_delay(),
+        )
+
+        self._deduplicator = Deduplicator(section.windows_paths())
+        self._transformer = Transformer(section.transform())
+
+        self._output_dir = OutputDirectory(
+            config.default_section.working_dir() / section.output_dir(name),
+            section.redownload(),
+            section.on_conflict(),
+        )
+
+    @property
+    def report(self) -> Report:
+        return self._output_dir.report
+
+    @property
+    def prev_report(self) -> Optional[Report]:
+        return self._output_dir.prev_report
+
+    @staticmethod
+    async def gather(awaitables: Sequence[Awaitable[Any]]) -> List[Any]:
+        """
+        Similar to asyncio.gather. However, in the case of an exception, all
+        still running tasks are cancelled and the exception is rethrown.
+
+        This should always be preferred over asyncio.gather in crawler code so
+        that an exception like CrawlError may actually stop the crawler.
+        """
+
+        tasks = [asyncio.ensure_future(aw) for aw in awaitables]
+        result = asyncio.gather(*tasks)
+        try:
+            return await result
+        except:  # noqa: E722
+            for task in tasks:
+                task.cancel()
+            raise
+
+    async def crawl(self, path: PurePath) -> Optional[CrawlToken]:
+        log.explain_topic(f"Decision: Crawl {fmt_path(path)}")
+        path = self._deduplicator.mark(path)
+        self._output_dir.report.found(path)
+
+        if self._transformer.transform(path) is None:
+            log.explain("Answer: No")
+            log.status("[bold bright_black]", "Ignored", fmt_path(path))
+            return None
+
+        log.explain("Answer: Yes")
+        return CrawlToken(self._limiter, path)
+
+    async def download(
+            self,
+            path: PurePath,
+            mtime: Optional[datetime] = None,
+            redownload: Optional[Redownload] = None,
+            on_conflict: Optional[OnConflict] = None,
+    ) -> Optional[DownloadToken]:
+        log.explain_topic(f"Decision: Download {fmt_path(path)}")
+        path = self._deduplicator.mark(path)
+        self._output_dir.report.found(path)
+
+        transformed_path = self._transformer.transform(path)
+        if transformed_path is None:
+            log.explain("Answer: No")
+            log.status("[bold bright_black]", "Ignored", fmt_path(path))
+            return None
+
+        fs_token = await self._output_dir.download(path, transformed_path, mtime, redownload, on_conflict)
+        if fs_token is None:
+            log.explain("Answer: No")
+            return None
+
+        log.explain("Answer: Yes")
+        return DownloadToken(self._limiter, fs_token, path)
+
+    async def _cleanup(self) -> None:
+        log.explain_topic("Decision: Clean up files")
+        if self.error_free:
+            log.explain("No warnings or errors occurred during this run")
+            log.explain("Answer: Yes")
+            await self._output_dir.cleanup()
+        else:
+            log.explain("Warnings or errors occurred during this run")
+            log.explain("Answer: No")
+
+    @anoncritical
+    async def run(self) -> None:
+        """
+        Start the crawling process. Call this function if you want to use a
+        crawler.
+        """
+
+        with log.show_progress():
+            self._output_dir.prepare()
+            self._output_dir.load_prev_report()
+            await self._run()
+            await self._cleanup()
+            self._output_dir.store_report()
+
+    @abstractmethod
+    async def _run(self) -> None:
+        """
+        Overwrite this function if you are writing a crawler.
+
+        This function must not return before all crawling is complete. To crawl
+        multiple things concurrently, asyncio.gather can be used.
+        """
+
+        pass
+
+    def debug_transforms(self) -> None:
+        self._output_dir.load_prev_report()
+
+        if not self.prev_report:
+            log.warn("Couldn't find or load old report")
+            return
+
+        seen: Set[PurePath] = set()
+        for known in sorted(self.prev_report.found_paths):
+            looking_at = list(reversed(known.parents)) + [known]
+            for path in looking_at:
+                if path in seen:
+                    continue
+
+                log.explain_topic(f"Transforming {fmt_path(path)}")
+                self._transformer.transform(path)
+                seen.add(path)

PFERD/crawl/http_crawler.py

@@ -0,0 +1,199 @@
+import asyncio
+import http.cookies
+import ssl
+from pathlib import Path, PurePath
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+import certifi
+from aiohttp.client import ClientTimeout
+
+from ..auth import Authenticator
+from ..config import Config
+from ..logging import log
+from ..utils import fmt_real_path
+from ..version import NAME, VERSION
+from .crawler import Crawler, CrawlerSection
+
+
+class HttpCrawlerSection(CrawlerSection):
+    def http_timeout(self) -> float:
+        return self.s.getfloat("http_timeout", fallback=20)
+
+
+class HttpCrawler(Crawler):
+    COOKIE_FILE = PurePath(".cookies")
+
+    def __init__(
+            self,
+            name: str,
+            section: HttpCrawlerSection,
+            config: Config,
+            shared_auth: Optional[Authenticator] = None,
+    ) -> None:
+        super().__init__(name, section, config)
+
+        self._authentication_id = 0
+        self._authentication_lock = asyncio.Lock()
+        self._request_count = 0
+        self._http_timeout = section.http_timeout()
+
+        self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
+        self._shared_cookie_jar_paths: Optional[List[Path]] = None
+        self._shared_auth = shared_auth
+
+        self._output_dir.register_reserved(self.COOKIE_FILE)
+
+    async def _current_auth_id(self) -> int:
+        """
+        Returns the id for the current authentication, i.e. an identifier for the last
+        successful call to [authenticate].
+
+        This method must be called before any request that might authenticate is made, so the
+        HttpCrawler can properly track when [authenticate] can return early and when actual
+        authentication is necessary.
+        """
+        # We acquire the lock here to ensure we wait for any concurrent authenticate to finish.
+        # This should reduce the amount of requests we make: If an authentication is in progress
+        # all future requests wait for authentication to complete.
+        async with self._authentication_lock:
+            self._request_count += 1
+            return self._authentication_id
+
+    async def authenticate(self, caller_auth_id: int) -> None:
+        """
+        Starts the authentication process. The main work is offloaded to _authenticate, which
+        you should overwrite in a subclass if needed. This method should *NOT* be overwritten.
+
+        The [caller_auth_id] should be the result of a [_current_auth_id] call made *before*
+        the request was made. This ensures that authentication is not performed needlessly.
+        """
+        async with self._authentication_lock:
+            log.explain_topic("Authenticating")
+            # Another thread successfully called authenticate in-between
+            # We do not want to perform auth again, so we return here. We can
+            # assume the other thread suceeded as authenticate will throw an error
+            # if it failed and aborts the crawl process.
+            if caller_auth_id != self._authentication_id:
+                log.explain(
+                    "Authentication skipped due to auth id mismatch."
+                    "A previous authentication beat us to the race."
+                )
+                return
+            log.explain("Calling crawler-specific authenticate")
+            await self._authenticate()
+            self._authentication_id += 1
+            # Saving the cookies after the first auth ensures we won't need to re-authenticate
+            # on the next run, should this one be aborted or crash
+            self._save_cookies()
+
+    async def _authenticate(self) -> None:
+        """
+        Performs authentication. This method must only return normally if authentication suceeded.
+        In all other cases it must either retry internally or throw a terminal exception.
+        """
+        raise RuntimeError("_authenticate() was called but crawler doesn't provide an implementation")
+
+    def share_cookies(self, shared: Dict[Authenticator, List[Path]]) -> None:
+        if not self._shared_auth:
+            return
+
+        if self._shared_auth in shared:
+            self._shared_cookie_jar_paths = shared[self._shared_auth]
+        else:
+            self._shared_cookie_jar_paths = []
+            shared[self._shared_auth] = self._shared_cookie_jar_paths
+
+        self._shared_cookie_jar_paths.append(self._cookie_jar_path)
+
+    def _load_cookies_from_file(self, path: Path) -> None:
+        jar: Any = http.cookies.SimpleCookie()
+        with open(path, encoding="utf-8") as f:
+            for i, line in enumerate(f):
+                # Names of headers are case insensitive
+                if line[:11].lower() == "set-cookie:":
+                    jar.load(line[11:])
+                else:
+                    log.explain(f"Line {i} doesn't start with 'Set-Cookie:', ignoring it")
+        self._cookie_jar.update_cookies(jar)
+
+    def _save_cookies_to_file(self, path: Path) -> None:
+        jar: Any = http.cookies.SimpleCookie()
+        for morsel in self._cookie_jar:
+            jar[morsel.key] = morsel
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(jar.output(sep="\n"))
+            f.write("\n")  # A trailing newline is just common courtesy
+
+    def _load_cookies(self) -> None:
+        log.explain_topic("Loading cookies")
+
+        cookie_jar_path: Optional[Path] = None
+
+        if self._shared_cookie_jar_paths is None:
+            log.explain("Not sharing any cookies")
+            cookie_jar_path = self._cookie_jar_path
+        else:
+            log.explain("Sharing cookies")
+            max_mtime: Optional[float] = None
+            for path in self._shared_cookie_jar_paths:
+                if not path.is_file():
+                    log.explain(f"{fmt_real_path(path)} is not a file")
+                    continue
+                mtime = path.stat().st_mtime
+                if max_mtime is None or mtime > max_mtime:
+                    log.explain(f"{fmt_real_path(path)} has newest mtime so far")
+                    max_mtime = mtime
+                    cookie_jar_path = path
+                else:
+                    log.explain(f"{fmt_real_path(path)} has older mtime")
+
+        if cookie_jar_path is None:
+            log.explain("Couldn't find a suitable cookie file")
+            return
+
+        log.explain(f"Loading cookies from {fmt_real_path(cookie_jar_path)}")
+        try:
+            self._load_cookies_from_file(cookie_jar_path)
+        except Exception as e:
+            log.explain("Failed to load cookies")
+            log.explain(str(e))
+
+    def _save_cookies(self) -> None:
+        log.explain_topic("Saving cookies")
+
+        try:
+            log.explain(f"Saving cookies to {fmt_real_path(self._cookie_jar_path)}")
+            self._save_cookies_to_file(self._cookie_jar_path)
+        except Exception as e:
+            log.warn(f"Failed to save cookies to {fmt_real_path(self._cookie_jar_path)}")
+            log.warn(str(e))
+
+    async def run(self) -> None:
+        self._request_count = 0
+        self._cookie_jar = aiohttp.CookieJar()
+        self._load_cookies()
+
+        async with aiohttp.ClientSession(
+                headers={"User-Agent": f"{NAME}/{VERSION}"},
+                cookie_jar=self._cookie_jar,
+                connector=aiohttp.TCPConnector(ssl=ssl.create_default_context(cafile=certifi.where())),
+                timeout=ClientTimeout(
+                    # 30 minutes. No download in the history of downloads was longer than 30 minutes.
+                    # This is enough to transfer a 600 MB file over a 3 Mib/s connection.
+                    # Allowing an arbitrary value could be annoying for overnight batch jobs
+                    total=15 * 60,
+                    connect=self._http_timeout,
+                    sock_connect=self._http_timeout,
+                    sock_read=self._http_timeout,
+                )
+        ) as session:
+            self.session = session
+            try:
+                await super().run()
+            finally:
+                del self.session
+        log.explain_topic(f"Total amount of HTTP requests: {self._request_count}")
+
+        # They are saved in authenticate, but a final save won't hurt
+        self._save_cookies()

PFERD/crawl/ilias/__init__.py

@@ -0,0 +1,3 @@
+from .kit_ilias_web_crawler import KitIliasWebCrawler, KitIliasWebCrawlerSection
+
+__all__ = ["KitIliasWebCrawler", "KitIliasWebCrawlerSection"]

PFERD/crawl/ilias/file_templates.py

@@ -0,0 +1,201 @@
+from enum import Enum
+from typing import Optional
+
+import bs4
+
+from PFERD.utils import soupify
+
+_link_template_plain = "{{link}}"
+_link_template_fancy = """
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>ILIAS - Link: {{name}}</title>
+        <meta http-equiv = "refresh" content = "{{redirect_delay}}; url = {{link}}" />
+    </head>
+
+    <style>
+    * {
+        box-sizing: border-box;
+    }
+    .center-flex {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+    }
+    body {
+        padding: 0;
+        margin: 0;
+        background-color: #f0f0f0;
+        font-family: "Open Sans", Verdana, Arial, Helvetica, sans-serif;
+        height: 100vh;
+    }
+    .row {
+        background-color: white;
+        min-width: 500px;
+        max-width: 90vw;
+        display: flex;
+        padding: 1em;
+    }
+    .logo {
+        flex: 0 1;
+        margin-right: 1em;
+        fill: #009682;
+    }
+    .tile {
+        flex: 1 0;
+        display: flex;
+        flex-direction: column;
+        justify-content: center;
+    }
+    .top-row {
+        padding-bottom: 5px;
+        font-size: 15px;
+    }
+    a {
+        color: #009682;
+        text-decoration: none;
+    }
+    a:hover {
+        text-decoration: underline;
+    }
+    .bottom-row {
+        font-size: 13px;
+    }
+    .menu-button {
+        border: 1px solid black;
+        margin-left: 4em;
+        width: 25px;
+        height: 25px;
+        flex: 0 0 25px;
+        background-color: #b3e0da;
+        font-size: 13px;
+        color: #222;
+    }
+    </style>
+    <body class="center-flex">
+        <div class="row">
+            <div class="logo center-flex">
+                <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
+                    <path d="M12 0c-6.627 0-12 5.373-12 12s5.373 12 12 12 12-5.373 12-12-5.373-12-12-12zm9.567 9.098c-.059-.058-.127-.108-.206-.138-.258-.101-1.35.603-1.515.256-.108-.231-.327.148-.578.008-.121-.067-.459-.52-.611-.465-.312.112.479.974.694 1.087.203-.154.86-.469 1.002-.039.271.812-.745 1.702-1.264 2.171-.775.702-.63-.454-1.159-.86-.277-.213-.274-.667-.555-.824-.125-.071-.7-.732-.694-.821l-.017.167c-.095.072-.297-.27-.319-.325 0 .298.485.772.646 1.011.273.409.42 1.005.756 1.339.179.18.866.923 1.045.908l.921-.437c.649.154-1.531 3.237-1.738 3.619-.171.321.139 1.112.114 1.49-.029.437-.374.579-.7.817-.35.255-.268.752-.562.934-.521.321-.897 1.366-1.639 1.361-.219-.001-1.151.364-1.273.007-.095-.258-.223-.455-.356-.71-.131-.25-.015-.51-.175-.731-.11-.154-.479-.502-.513-.684-.002-.157.118-.632.283-.715.231-.118.044-.462.016-.663-.048-.357-.27-.652-.535-.859-.393-.302-.189-.542-.098-.974 0-.206-.126-.476-.402-.396-.57.166-.396-.445-.812-.417-.299.021-.543.211-.821.295-.349.104-.707-.083-1.053-.126-1.421-.179-1.885-1.804-1.514-2.976.037-.192-.115-.547-.048-.696.159-.352.485-.752.768-1.021.16-.152.365-.113.553-.231.29-.182.294-.558.578-.789.404-.328.956-.321 1.482-.392.281-.037 1.35-.268 1.518-.06 0 .039.193.611-.019.578.438.023 1.061.756 1.476.585.213-.089.135-.744.573-.427.265.19 1.45.275 1.696.07.152-.125.236-.939.053-1.031.117.116-.618.125-.686.099-.122-.044-.235.115-.43.025.117.055-.651-.358-.22-.674-.181.132-.349-.037-.544.109-.135.109.062.181-.13.277-.305.155-.535-.53-.649-.607-.118-.077-1.024-.713-.777-.298l.797.793c-.04.026-.209-.289-.209-.059.053-.136.02.585-.105.35-.056-.09.091-.14.006-.271 0-.085-.23-.169-.275-.228-.126-.157-.462-.502-.644-.585-.05-.024-.771.088-.832.111-.071.099-.131.203-.181.314-.149.055-.29.127-.423.216l-.159.356c-.068.061-.772.294-.776.303.03-.076-.492-.172-.457-.324.038-.167.215-.687.169-.877-.048-.199 1.085.287 1.158-.238.029-.227.047-.492-.316-.531.069.008.702-.249.807-.364.148-.169.486-.447.731-.447.286 0 .225-.417.356-.622.133.053-.071.38.088.512-.01-.104.45.057.494.033.105-.056.691-.023.601-.299-.101-.28.052-.197.183-.255-.02.008.248-.458.363-.456-.104-.089-.398.112-.516.103-.308-.024-.177-.525-.061-.672.09-.116-.246-.258-.25-.036-.006.332-.314.633-.243 1.075.109.666-.743-.161-.816-.115-.283.172-.515-.216-.368-.449.149-.238.51-.226.659-.48.104-.179.227-.389.388-.524.541-.454.689-.091 1.229-.042.526.048.178.125.105.327-.07.192.289.261.413.1.071-.092.232-.326.301-.499.07-.175.578-.2.527-.365 2.72 1.148 4.827 3.465 5.694 6.318zm-11.113-3.779l.068-.087.073-.019c.042-.034.086-.118.151-.104.043.009.146.095.111.148-.037.054-.066-.049-.081.101-.018.169-.188.167-.313.222-.087.037-.175-.018-.09-.104l.088-.108-.007-.049zm.442.245c.046-.045.138-.008.151-.094.014-.084.078-.178-.008-.335-.022-.042.116-.082.051-.137l-.109.032s.155-.668.364-.366l-.089.103c.135.134.172.47.215.687.127.066.324.078.098.192.117-.02-.618.314-.715.178-.072-.083.317-.139.307-.173-.004-.011-.317-.02-.265-.087zm1.43-3.547l-.356.326c-.36.298-1.28.883-1.793.705-.524-.18-1.647.667-1.826.673-.067.003.002-.641.36-.689-.141.021.993-.575 1.185-.805.678-.146 1.381-.227 2.104-.227l.326.017zm-5.086 1.19c.07.082.278.092-.026.288-.183.11-.377.809-.548.809-.51.223-.542-.439-1.109.413-.078.115-.395.158-.644.236.685-.688 1.468-1.279 2.327-1.746zm-5.24 8.793c0-.541.055-1.068.139-1.586l.292.185c.113.135.113.719.169.911.139.482.484.751.748 1.19.155.261.414.923.332 1.197.109-.179 1.081.824 1.259 1.033.418.492.74 1.088.061 1.574-.219.158.334 1.14.049 1.382l-.365.094c-.225.138-.235.397-.166.631-1.562-1.765-2.518-4.076-2.518-6.611zm14.347-5.823c.083-.01-.107.167-.107.167.033.256.222.396.581.527.437.157.038.455-.213.385-.139-.039-.854-.255-.879.025 0 .167-.679.001-.573-.175.073-.119.05-.387.186-.562.193-.255.38-.116.386.032-.001.394.398-.373.619-.399z"/>
+                </svg>
+            </div>
+            <div class="tile">
+                <div class="top-row">
+                    <a href="{{link}}">{{name}}</a>
+                </div>
+                <div class="bottom-row">{{description}}</div>
+            </div>
+            <div class="menu-button center-flex"> ⯆ </div>
+        </div>
+    </body>
+</html>
+""".strip()  # noqa: E501 line too long
+
+_link_template_internet_shortcut = """
+[InternetShortcut]
+URL={{link}}
+""".strip()
+
+_learning_module_template = """
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>{{name}}</title>
+    </head>
+
+    <style>
+    * {
+        box-sizing: border-box;
+    }
+    .center-flex {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+    }
+    .nav {
+        display: flex;
+        justify-content: space-between;
+    }
+    </style>
+    <body class="center-flex">
+{{body}}
+    </body>
+</html>
+"""
+
+
+def learning_module_template(body: bs4.Tag, name: str, prev: Optional[str], next: Optional[str]) -> str:
+    # Seems to be comments, ignore those.
+    for elem in body.select(".il-copg-mob-fullscreen-modal"):
+        elem.decompose()
+
+    nav_template = """
+        <div class="nav">
+            {{left}}
+            {{right}}
+        </div>
+    """
+    if prev and body.select_one(".ilc_page_lnav_LeftNavigation"):
+        text = body.select_one(".ilc_page_lnav_LeftNavigation").getText().strip()
+        left = f'<a href="{prev}">{text}</a>'
+    else:
+        left = "<span></span>"
+
+    if next and body.select_one(".ilc_page_rnav_RightNavigation"):
+        text = body.select_one(".ilc_page_rnav_RightNavigation").getText().strip()
+        right = f'<a href="{next}">{text}</a>'
+    else:
+        right = "<span></span>"
+
+    if top_nav := body.select_one(".ilc_page_tnav_TopNavigation"):
+        top_nav.replace_with(
+            soupify(nav_template.replace("{{left}}", left).replace("{{right}}", right).encode())
+        )
+
+    if bot_nav := body.select_one(".ilc_page_bnav_BottomNavigation"):
+        bot_nav.replace_with(soupify(nav_template.replace(
+            "{{left}}", left).replace("{{right}}", right).encode())
+        )
+
+    body = body.prettify()
+    return _learning_module_template.replace("{{body}}", body).replace("{{name}}", name)
+
+
+class Links(Enum):
+    IGNORE = "ignore"
+    PLAINTEXT = "plaintext"
+    FANCY = "fancy"
+    INTERNET_SHORTCUT = "internet-shortcut"
+
+    def template(self) -> Optional[str]:
+        if self == Links.FANCY:
+            return _link_template_fancy
+        elif self == Links.PLAINTEXT:
+            return _link_template_plain
+        elif self == Links.INTERNET_SHORTCUT:
+            return _link_template_internet_shortcut
+        elif self == Links.IGNORE:
+            return None
+        raise ValueError("Missing switch case")
+
+    def extension(self) -> Optional[str]:
+        if self == Links.FANCY:
+            return ".html"
+        elif self == Links.PLAINTEXT:
+            return ".txt"
+        elif self == Links.INTERNET_SHORTCUT:
+            return ".url"
+        elif self == Links.IGNORE:
+            return None
+        raise ValueError("Missing switch case")
+
+    @staticmethod
+    def from_string(string: str) -> "Links":
+        try:
+            return Links(string)
+        except ValueError:
+            raise ValueError("must be one of 'ignore', 'plaintext',"
+                             " 'html', 'internet-shortcut'")

PFERD/crawl/ilias/ilias_html_cleaner.py

@@ -0,0 +1,91 @@
+from bs4 import BeautifulSoup, Comment, Tag
+
+_STYLE_TAG_CONTENT = """
+    .ilc_text_block_Information {
+      background-color: #f5f7fa;
+    }
+    div.ilc_text_block_Standard {
+      margin-bottom: 10px;
+      margin-top: 10px;
+    }
+    span.ilc_text_inline_Strong {
+      font-weight: bold;
+    }
+
+    .accordion-head {
+      background-color: #f5f7fa;
+      padding: 0.5rem 0;
+    }
+
+    h3 {
+      margin-top: 0.5rem;
+      margin-bottom: 1rem;
+    }
+
+    br.visible-break {
+      margin-bottom: 1rem;
+    }
+
+    article {
+      margin: 0.5rem 0;
+    }
+
+    body {
+      padding: 1em;
+      grid-template-columns: 1fr min(60rem, 90%) 1fr;
+      line-height: 1.2;
+    }
+"""
+
+_ARTICLE_WORTHY_CLASSES = [
+    "ilc_text_block_Information",
+    "ilc_section_Attention",
+    "ilc_section_Link",
+]
+
+
+def insert_base_markup(soup: BeautifulSoup) -> BeautifulSoup:
+    head = soup.new_tag("head")
+    soup.insert(0, head)
+
+    simplecss_link: Tag = soup.new_tag("link")
+    # <link rel="stylesheet" href="https://cdn.simplecss.org/simple.css">
+    simplecss_link["rel"] = "stylesheet"
+    simplecss_link["href"] = "https://cdn.simplecss.org/simple.css"
+    head.append(simplecss_link)
+
+    # Basic style tags for compat
+    style: Tag = soup.new_tag("style")
+    style.append(_STYLE_TAG_CONTENT)
+    head.append(style)
+
+    return soup
+
+
+def clean(soup: BeautifulSoup) -> BeautifulSoup:
+    for block in soup.find_all(class_=lambda x: x in _ARTICLE_WORTHY_CLASSES):
+        block.name = "article"
+
+    for block in soup.find_all("h3"):
+        block.name = "div"
+
+    for block in soup.find_all("h1"):
+        block.name = "h3"
+
+    for block in soup.find_all(class_="ilc_va_ihcap_VAccordIHeadCap"):
+        block.name = "h3"
+        block["class"] += ["accordion-head"]
+
+    for dummy in soup.select(".ilc_text_block_Standard.ilc_Paragraph"):
+        children = list(dummy.children)
+        if not children:
+            dummy.decompose()
+        if len(children) > 1:
+            continue
+        if isinstance(type(children[0]), Comment):
+            dummy.decompose()
+
+    for hrule_imposter in soup.find_all(class_="ilc_section_Separator"):
+        hrule_imposter.insert(0, soup.new_tag("hr"))
+
+    return soup

PFERD/crawl/kit_ipd_crawler.py

@@ -0,0 +1,170 @@
+import os
+import re
+from dataclasses import dataclass
+from pathlib import PurePath
+from typing import Awaitable, List, Optional, Pattern, Set, Tuple, Union
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup, Tag
+
+from ..config import Config
+from ..logging import ProgressBar, log
+from ..output_dir import FileSink
+from ..utils import soupify
+from .crawler import CrawlError
+from .http_crawler import HttpCrawler, HttpCrawlerSection
+
+
+class KitIpdCrawlerSection(HttpCrawlerSection):
+    def target(self) -> str:
+        target = self.s.get("target")
+        if not target:
+            self.missing_value("target")
+
+        if not target.startswith("https://"):
+            self.invalid_value("target", target, "Should be a URL")
+
+        return target
+
+    def link_regex(self) -> Pattern[str]:
+        regex = self.s.get("link_regex", r"^.*?[^/]+\.(pdf|zip|c|cpp|java)$")
+        return re.compile(regex)
+
+
+@dataclass(unsafe_hash=True)
+class KitIpdFile:
+    name: str
+    url: str
+
+
+@dataclass
+class KitIpdFolder:
+    name: str
+    files: List[KitIpdFile]
+
+    def explain(self) -> None:
+        log.explain_topic(f"Folder {self.name!r}")
+        for file in self.files:
+            log.explain(f"File {file.name!r} (href={file.url!r})")
+
+    def __hash__(self) -> int:
+        return self.name.__hash__()
+
+
+class KitIpdCrawler(HttpCrawler):
+
+    def __init__(
+            self,
+            name: str,
+            section: KitIpdCrawlerSection,
+            config: Config,
+    ):
+        super().__init__(name, section, config)
+        self._url = section.target()
+        self._file_regex = section.link_regex()
+
+    async def _run(self) -> None:
+        maybe_cl = await self.crawl(PurePath("."))
+        if not maybe_cl:
+            return
+
+        tasks: List[Awaitable[None]] = []
+
+        async with maybe_cl:
+            for item in await self._fetch_items():
+                if isinstance(item, KitIpdFolder):
+                    tasks.append(self._crawl_folder(item))
+                else:
+                    # Orphan files are placed in the root folder
+                    tasks.append(self._download_file(PurePath("."), item))
+
+        await self.gather(tasks)
+
+    async def _crawl_folder(self, folder: KitIpdFolder) -> None:
+        path = PurePath(folder.name)
+        if not await self.crawl(path):
+            return
+
+        tasks = [self._download_file(path, file) for file in folder.files]
+
+        await self.gather(tasks)
+
+    async def _download_file(self, parent: PurePath, file: KitIpdFile) -> None:
+        element_path = parent / file.name
+        maybe_dl = await self.download(element_path)
+        if not maybe_dl:
+            return
+
+        async with maybe_dl as (bar, sink):
+            await self._stream_from_url(file.url, sink, bar)
+
+    async def _fetch_items(self) -> Set[Union[KitIpdFile, KitIpdFolder]]:
+        page, url = await self.get_page()
+        elements: List[Tag] = self._find_file_links(page)
+        items: Set[Union[KitIpdFile, KitIpdFolder]] = set()
+
+        for element in elements:
+            folder_label = self._find_folder_label(element)
+            if folder_label:
+                folder = self._extract_folder(folder_label, url)
+                if folder not in items:
+                    items.add(folder)
+                    folder.explain()
+            else:
+                file = self._extract_file(element, url)
+                items.add(file)
+                log.explain_topic(f"Orphan file {file.name!r} (href={file.url!r})")
+                log.explain("Attributing it to root folder")
+
+        return items
+
+    def _extract_folder(self, folder_tag: Tag, url: str) -> KitIpdFolder:
+        files: List[KitIpdFile] = []
+        name = folder_tag.getText().strip()
+
+        container: Tag = folder_tag.findNextSibling(name="table")
+        for link in self._find_file_links(container):
+            files.append(self._extract_file(link, url))
+
+        return KitIpdFolder(name, files)
+
+    @staticmethod
+    def _find_folder_label(file_link: Tag) -> Optional[Tag]:
+        enclosing_table: Tag = file_link.findParent(name="table")
+        if enclosing_table is None:
+            return None
+        return enclosing_table.findPreviousSibling(name=re.compile("^h[1-6]$"))
+
+    def _extract_file(self, link: Tag, url: str) -> KitIpdFile:
+        url = self._abs_url_from_link(url, link)
+        name = os.path.basename(url)
+        return KitIpdFile(name, url)
+
+    def _find_file_links(self, tag: Union[Tag, BeautifulSoup]) -> List[Tag]:
+        return tag.findAll(name="a", attrs={"href": self._file_regex})
+
+    def _abs_url_from_link(self, url: str, link_tag: Tag) -> str:
+        return urljoin(url, link_tag.get("href"))
+
+    async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar) -> None:
+        async with self.session.get(url, allow_redirects=False) as resp:
+            if resp.status == 403:
+                raise CrawlError("Received a 403. Are you within the KIT network/VPN?")
+            if resp.content_length:
+                bar.set_total(resp.content_length)
+
+            async for data in resp.content.iter_chunked(1024):
+                sink.file.write(data)
+                bar.advance(len(data))
+
+            sink.done()
+
+    async def get_page(self) -> Tuple[BeautifulSoup, str]:
+        async with self.session.get(self._url) as request:
+            # The web page for Algorithmen für Routenplanung contains some
+            # weird comments that beautifulsoup doesn't parse correctly. This
+            # hack enables those pages to be crawled, and should hopefully not
+            # cause issues on other pages.
+            content = (await request.read()).decode("utf-8")
+            content = re.sub(r"<!--.*?-->", "", content)
+            return soupify(content.encode("utf-8")), str(request.url)

PFERD/crawl/local_crawler.py

@@ -0,0 +1,117 @@
+import asyncio
+import datetime
+import random
+from pathlib import Path, PurePath
+from typing import Optional
+
+from ..config import Config
+from .crawler import Crawler, CrawlerSection, anoncritical
+
+
+class LocalCrawlerSection(CrawlerSection):
+    def target(self) -> Path:
+        value = self.s.get("target")
+        if value is None:
+            self.missing_value("target")
+        return Path(value).expanduser()
+
+    def crawl_delay(self) -> float:
+        value = self.s.getfloat("crawl_delay", fallback=0.0)
+        if value < 0:
+            self.invalid_value("crawl_delay", value,
+                               "Must not be negative")
+        return value
+
+    def download_delay(self) -> float:
+        value = self.s.getfloat("download_delay", fallback=0.0)
+        if value < 0:
+            self.invalid_value("download_delay", value,
+                               "Must not be negative")
+        return value
+
+    def download_speed(self) -> Optional[int]:
+        value = self.s.getint("download_speed")
+        if value is not None and value <= 0:
+            self.invalid_value("download_speed", value,
+                               "Must be greater than 0")
+        return value
+
+
+class LocalCrawler(Crawler):
+    def __init__(
+            self,
+            name: str,
+            section: LocalCrawlerSection,
+            config: Config,
+    ):
+        super().__init__(name, section, config)
+
+        self._target = config.default_section.working_dir() / section.target()
+        self._crawl_delay = section.crawl_delay()
+        self._download_delay = section.download_delay()
+        self._download_speed = section.download_speed()
+
+        if self._download_speed:
+            self._block_size = self._download_speed // 10
+        else:
+            self._block_size = 1024**2  # 1 MiB
+
+    async def _run(self) -> None:
+        await self._crawl_path(self._target, PurePath())
+
+    @anoncritical
+    async def _crawl_path(self, path: Path, pure: PurePath) -> None:
+        if path.is_dir():
+            await self._crawl_dir(path, pure)
+        elif path.is_file():
+            await self._crawl_file(path, pure)
+
+    async def _crawl_dir(self, path: Path, pure: PurePath) -> None:
+        cl = await self.crawl(pure)
+        if not cl:
+            return
+
+        tasks = []
+
+        async with cl:
+            await asyncio.sleep(random.uniform(
+                0.5 * self._crawl_delay,
+                self._crawl_delay,
+            ))
+
+            for child in path.iterdir():
+                pure_child = cl.path / child.name
+                tasks.append(self._crawl_path(child, pure_child))
+
+        await self.gather(tasks)
+
+    async def _crawl_file(self, path: Path, pure: PurePath) -> None:
+        stat = path.stat()
+        mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
+        dl = await self.download(pure, mtime=mtime)
+        if not dl:
+            return
+
+        async with dl as (bar, sink):
+            await asyncio.sleep(random.uniform(
+                0.5 * self._download_delay,
+                self._download_delay,
+            ))
+
+            bar.set_total(stat.st_size)
+
+            with open(path, "rb") as f:
+                while True:
+                    data = f.read(self._block_size)
+                    if len(data) == 0:
+                        break
+
+                    sink.file.write(data)
+                    bar.advance(len(data))
+
+                    if self._download_speed:
+                        delay = self._block_size / self._download_speed
+                        delay = random.uniform(0.8 * delay, 1.2 * delay)
+                        await asyncio.sleep(delay)
+
+                sink.done()

PFERD/deduplicator.py

@@ -0,0 +1,85 @@
+from pathlib import PurePath
+from typing import Iterator, Set
+
+from .logging import log
+from .utils import fmt_path
+
+
+def name_variants(path: PurePath) -> Iterator[PurePath]:
+    separator = " " if " " in path.stem else "_"
+    i = 1
+    while True:
+        yield path.parent / f"{path.stem}{separator}{i}{path.suffix}"
+        i += 1
+
+
+class Deduplicator:
+    FORBIDDEN_CHARS = '<>:"/\\|?*' + "".join([chr(i) for i in range(0, 32)])
+    FORBIDDEN_NAMES = {
+        "CON", "PRN", "AUX", "NUL",
+        "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
+        "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
+    }
+
+    def __init__(self, windows_paths: bool) -> None:
+        self._windows_paths = windows_paths
+
+        self._known: Set[PurePath] = set()
+
+    def _add(self, path: PurePath) -> None:
+        self._known.add(path)
+
+        # The last parent is just "."
+        for parent in list(path.parents)[:-1]:
+            self._known.add(parent)
+
+    def _fixup_element(self, name: str) -> str:
+        # For historical reasons, windows paths have some odd restrictions that
+        # we're trying to avoid. See:
+        # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+
+        for char in self.FORBIDDEN_CHARS:
+            name = name.replace(char, "_")
+
+        path = PurePath(name)
+        if path.stem in self.FORBIDDEN_NAMES:
+            name = f"{path.stem}_{path.suffix}"
+
+        if name.endswith(" ") or name.endswith("."):
+            name += "_"
+
+        return name
+
+    def _fixup_for_windows(self, path: PurePath) -> PurePath:
+        new_path = PurePath(*[self._fixup_element(elem) for elem in path.parts])
+        if new_path != path:
+            log.explain(f"Changed path to {fmt_path(new_path)} for windows compatibility")
+        return new_path
+
+    def fixup_path(self, path: PurePath) -> PurePath:
+        """Fixes up the path for windows, if enabled. Returns the path unchanged otherwise."""
+        if self._windows_paths:
+            return self._fixup_for_windows(path)
+        return path
+
+    def mark(self, path: PurePath) -> PurePath:
+        if self._windows_paths:
+            path = self._fixup_for_windows(path)
+
+        if path not in self._known:
+            self._add(path)
+            return path
+
+        log.explain(f"Path {fmt_path(path)} is already taken, finding a new name")
+
+        for variant in name_variants(path):
+            if variant in self._known:
+                log.explain(f"Path {fmt_path(variant)} is taken as well")
+                continue
+
+            log.explain(f"Found unused path {fmt_path(variant)}")
+            self._add(variant)
+            return variant
+
+        # The "name_variants" iterator returns infinitely many paths
+        raise RuntimeError("Unreachable")

PFERD/diva.py

@@ -1,169 +0,0 @@
-"""
-Utility functions and a scraper/downloader for the KIT DIVA portal.
-"""
-import logging
-import re
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Callable, List, Optional
-
-import requests
-
-from .errors import FatalException
-from .logging import PrettyLogger
-from .organizer import Organizer
-from .tmp_dir import TmpDir
-from .transform import Transformable
-from .utils import stream_to_path
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-@dataclass
-class DivaDownloadInfo(Transformable):
-    """
-    Information about a DIVA video
-    """
-    url: str
-
-
-DivaDownloadStrategy = Callable[[Organizer, DivaDownloadInfo], bool]
-
-
-def diva_download_new(organizer: Organizer, info: DivaDownloadInfo) -> bool:
-    """
-    Accepts only new files.
-    """
-    resolved_file = organizer.resolve(info.path)
-    if not resolved_file.exists():
-        return True
-    PRETTY.ignored_file(info.path, "local file exists")
-    return False
-
-
-class DivaPlaylistCrawler:
-    # pylint: disable=too-few-public-methods
-    """
-    A crawler for DIVA playlists.
-    """
-
-    _PLAYLIST_BASE_URL = "https://mediaservice.bibliothek.kit.edu/asset/detail/"
-    _COLLECTION_BASE_URL = "https://mediaservice.bibliothek.kit.edu/asset/collection.json"
-
-    def __init__(self, playlist_id: str):
-        self._id = playlist_id
-
-    @classmethod
-    def fetch_id(cls, playlist_link: str) -> str:
-        """
-        Fetches the ID for a playerlist, given the base link
-        (e.g. https://mediaservice.bibliothek.kit.edu/#/details/DIVA-2019-271).
-
-        Raises a FatalException, if the id can not be resolved
-        """
-        match = re.match(r".+#/details/(.+)", playlist_link)
-        if match is None:
-            raise FatalException(
-                "DIVA: Invalid playlist link format, could not extract details."
-            )
-        base_name = match.group(1)
-
-        response = requests.get(cls._PLAYLIST_BASE_URL + base_name + ".json")
-
-        if response.status_code != 200:
-            raise FatalException(
-                f"DIVA: Got non-200 status code ({response.status_code}))"
-                f"when requesting {response.url!r}!"
-            )
-
-        body = response.json()
-
-        if body["error"]:
-            raise FatalException(f"DIVA: Server returned error {body['error']!r}.")
-
-        return body["result"]["collection"]["id"]
-
-    def crawl(self) -> List[DivaDownloadInfo]:
-        """
-        Crawls the playlist given in the constructor.
-        """
-        response = requests.get(self._COLLECTION_BASE_URL, params={"collection": self._id})
-        if response.status_code != 200:
-            raise FatalException(f"Server returned status {response.status_code}.")
-
-        body = response.json()
-
-        if body["error"]:
-            raise FatalException(f"Server returned error {body['error']!r}.")
-
-        result = body["result"]
-
-        if result["resultCount"] > result["pageSize"]:
-            PRETTY.warning("Did not receive all results, some will be missing")
-
-        download_infos: List[DivaDownloadInfo] = []
-
-        for video in result["resultList"]:
-            title = video["title"]
-            collection_title = self._follow_path(["collection", "title"], video)
-            url = self._follow_path(
-                ["resourceList", "derivateList", "mp4", "url"],
-                video
-            )
-
-            if url and collection_title and title:
-                path = Path(collection_title, title + ".mp4")
-                download_infos.append(DivaDownloadInfo(path, url))
-            else:
-                PRETTY.warning(f"Incomplete video found: {title!r} {collection_title!r} {url!r}")
-
-        return download_infos
-
-    @staticmethod
-    def _follow_path(path: List[str], obj: Any) -> Optional[Any]:
-        """
-        Follows a property path through an object, bailing at the first None.
-        """
-        current = obj
-        for path_step in path:
-            if path_step in current:
-                current = current[path_step]
-            else:
-                return None
-        return current
-
-
-class DivaDownloader:
-    """
-    A downloader for DIVA videos.
-    """
-
-    def __init__(self, tmp_dir: TmpDir, organizer: Organizer, strategy: DivaDownloadStrategy):
-        self._tmp_dir = tmp_dir
-        self._organizer = organizer
-        self._strategy = strategy
-        self._session = requests.session()
-
-    def download_all(self, infos: List[DivaDownloadInfo]) -> None:
-        """
-        Download multiple files one after the other.
-        """
-        for info in infos:
-            self.download(info)
-
-    def download(self, info: DivaDownloadInfo) -> None:
-        """
-        Download a single file.
-        """
-        if not self._strategy(self._organizer, info):
-            self._organizer.mark(info.path)
-            return
-
-        with self._session.get(info.url, stream=True) as response:
-            if response.status_code == 200:
-                tmp_file = self._tmp_dir.new_path()
-                stream_to_path(response, tmp_file, info.path.name)
-                self._organizer.accept_file(tmp_file, info.path)
-            else:
-                PRETTY.warning(f"Could not download file, got response {response.status_code}")

PFERD/download_summary.py

@@ -1,69 +0,0 @@
-"""
-Provides a summary that keeps track of new modified or deleted files.
-"""
-from pathlib import Path
-from typing import List
-
-
-class DownloadSummary:
-    """
-    Keeps track of all new, modified or deleted files and provides a summary.
-    """
-
-    def __init__(self) -> None:
-        self._new_files: List[Path] = []
-        self._modified_files: List[Path] = []
-        self._deleted_files: List[Path] = []
-
-    @property
-    def new_files(self) -> List[Path]:
-        """
-        Returns all new files.
-        """
-        return self._new_files.copy()
-
-    @property
-    def modified_files(self) -> List[Path]:
-        """
-        Returns all modified files.
-        """
-        return self._modified_files.copy()
-
-    @property
-    def deleted_files(self) -> List[Path]:
-        """
-        Returns all deleted files.
-        """
-        return self._deleted_files.copy()
-
-    def merge(self, summary: 'DownloadSummary') -> None:
-        """
-        Merges ourselves with the passed summary. Modifies this object, but not the passed one.
-        """
-        self._new_files += summary.new_files
-        self._modified_files += summary.modified_files
-        self._deleted_files += summary.deleted_files
-
-    def add_deleted_file(self, path: Path) -> None:
-        """
-        Registers a file as deleted.
-        """
-        self._deleted_files.append(path)
-
-    def add_modified_file(self, path: Path) -> None:
-        """
-        Registers a file as changed.
-        """
-        self._modified_files.append(path)
-
-    def add_new_file(self, path: Path) -> None:
-        """
-        Registers a file as new.
-        """
-        self._new_files.append(path)
-
-    def has_updates(self) -> bool:
-        """
-        Returns whether this summary has any updates.
-        """
-        return bool(self._new_files or self._modified_files or self._deleted_files)

PFERD/downloaders.py

@@ -1,72 +0,0 @@
-"""
-General downloaders useful in many situations
-"""
-
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
-
-import requests
-import requests.auth
-
-from .organizer import Organizer
-from .tmp_dir import TmpDir
-from .transform import Transformable
-from .utils import stream_to_path
-
-
-@dataclass
-class HttpDownloadInfo(Transformable):
-    """
-    This class describes a single file to be downloaded.
-    """
-
-    url: str
-    parameters: Dict[str, Any] = field(default_factory=dict)
-
-
-class HttpDownloader:
-    """A HTTP downloader that can handle HTTP basic auth."""
-
-    def __init__(
-            self,
-            tmp_dir: TmpDir,
-            organizer: Organizer,
-            username: Optional[str],
-            password: Optional[str],
-    ):
-        """Create a new http downloader."""
-        self._organizer = organizer
-        self._tmp_dir = tmp_dir
-        self._username = username
-        self._password = password
-        self._session = self._build_session()
-
-    def _build_session(self) -> requests.Session:
-        session = requests.Session()
-        if self._username and self._password:
-            session.auth = requests.auth.HTTPBasicAuth(
-                self._username, self._password
-            )
-        return session
-
-    def download_all(self, infos: List[HttpDownloadInfo]) -> None:
-        """
-        Download multiple files one after the other.
-        """
-
-        for info in infos:
-            self.download(info)
-
-    def download(self, info: HttpDownloadInfo) -> None:
-        """
-        Download a single file.
-        """
-
-        with self._session.get(info.url, params=info.parameters, stream=True) as response:
-            if response.status_code == 200:
-                tmp_file = self._tmp_dir.new_path()
-                stream_to_path(response, tmp_file, info.path.name)
-                self._organizer.accept_file(tmp_file, info.path)
-            else:
-                # TODO use proper exception
-                raise Exception(f"Could not download file, got response {response.status_code}")

PFERD/errors.py

@@ -1,39 +0,0 @@
-"""
-An error logging decorator.
-"""
-
-import logging
-from typing import Any, Callable, TypeVar, cast
-
-from rich.console import Console
-
-from .logging import PrettyLogger
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-class FatalException(Exception):
-    """
-    A fatal exception occurred. Recovery is not possible.
-    """
-
-
-TFun = TypeVar('TFun', bound=Callable[..., Any])
-
-
-def swallow_and_print_errors(function: TFun) -> TFun:
-    """
-    Decorates a function, swallows all errors, logs them and returns none if one occurred.
-    """
-    def inner(*args: Any, **kwargs: Any) -> Any:
-        # pylint: disable=broad-except
-        try:
-            return function(*args, **kwargs)
-        except FatalException as error:
-            PRETTY.error(str(error))
-            return None
-        except Exception as error:
-            Console().print_exception()
-            return None
-    return cast(TFun, inner)

PFERD/ilias/__init__.py

@@ -1,10 +0,0 @@
-"""
-Synchronizing files from ILIAS instances (https://www.ilias.de/).
-"""
-
-from .authenticators import IliasAuthenticator, KitShibbolethAuthenticator
-from .crawler import (IliasCrawler, IliasCrawlerEntry, IliasDirectoryFilter,
-                      IliasElementType)
-from .downloader import (IliasDownloader, IliasDownloadInfo,
-                         IliasDownloadStrategy, download_everything,
-                         download_modified_or_new)

PFERD/ilias/authenticators.py

@@ -1,131 +0,0 @@
-"""
-Authenticators that can obtain proper ILIAS session cookies.
-"""
-
-import abc
-import logging
-from typing import Optional
-
-import bs4
-import requests
-
-from ..authenticators import TfaAuthenticator, UserPassAuthenticator
-from ..utils import soupify
-
-LOGGER = logging.getLogger(__name__)
-
-
-class IliasAuthenticator(abc.ABC):
-    # pylint: disable=too-few-public-methods
-
-    """
-    An authenticator that logs an existing requests session into an ILIAS
-    account.
-    """
-
-    @abc.abstractmethod
-    def authenticate(self, sess: requests.Session) -> None:
-        """
-        Log a requests session into this authenticator's ILIAS account.
-        """
-
-
-class KitShibbolethAuthenticator(IliasAuthenticator):
-    # pylint: disable=too-few-public-methods
-
-    """
-    Authenticate via KIT's shibboleth system.
-    """
-
-    def __init__(self, username: Optional[str] = None, password: Optional[str] = None) -> None:
-        self._auth = UserPassAuthenticator("KIT ILIAS Shibboleth", username, password)
-        self._tfa_auth = TfaAuthenticator("KIT ILIAS Shibboleth")
-
-    def authenticate(self, sess: requests.Session) -> None:
-        """
-        Performs the ILIAS Shibboleth authentication dance and saves the login
-        cookies it receieves.
-
-        This function should only be called whenever it is detected that you're
-        not logged in. The cookies obtained should be good for a few minutes,
-        maybe even an hour or two.
-        """
-
-        # Equivalent: Click on "Mit KIT-Account anmelden" button in
-        # https://ilias.studium.kit.edu/login.php
-        LOGGER.debug("Begin authentication process with ILIAS")
-        url = "https://ilias.studium.kit.edu/Shibboleth.sso/Login"
-        data = {
-            "sendLogin": "1",
-            "idp_selection": "https://idp.scc.kit.edu/idp/shibboleth",
-            "target": "/shib_login.php",
-            "home_organization_selection": "Mit KIT-Account anmelden",
-        }
-        soup = soupify(sess.post(url, data=data))
-
-        # Attempt to login using credentials, if necessary
-        while not self._login_successful(soup):
-            # Searching the form here so that this fails before asking for
-            # credentials rather than after asking.
-            form = soup.find("form", {"class": "full content", "method": "post"})
-            action = form["action"]
-
-            # Equivalent: Enter credentials in
-            # https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
-            LOGGER.debug("Attempt to log in to Shibboleth using credentials")
-            url = "https://idp.scc.kit.edu" + action
-            data = {
-                "_eventId_proceed": "",
-                "j_username": self._auth.username,
-                "j_password": self._auth.password,
-            }
-            soup = soupify(sess.post(url, data=data))
-
-            if self._tfa_required(soup):
-                soup = self._authenticate_tfa(sess, soup)
-
-            if not self._login_successful(soup):
-                print("Incorrect credentials.")
-                self._auth.invalidate_credentials()
-
-        # Equivalent: Being redirected via JS automatically
-        # (or clicking "Continue" if you have JS disabled)
-        LOGGER.debug("Redirect back to ILIAS with login information")
-        relay_state = soup.find("input", {"name": "RelayState"})
-        saml_response = soup.find("input", {"name": "SAMLResponse"})
-        url = "https://ilias.studium.kit.edu/Shibboleth.sso/SAML2/POST"
-        data = {  # using the info obtained in the while loop above
-            "RelayState": relay_state["value"],
-            "SAMLResponse": saml_response["value"],
-        }
-        sess.post(url, data=data)
-
-    def _authenticate_tfa(
-            self,
-            session: requests.Session,
-            soup: bs4.BeautifulSoup
-    ) -> bs4.BeautifulSoup:
-        # Searching the form here so that this fails before asking for
-        # credentials rather than after asking.
-        form = soup.find("form", {"method": "post"})
-        action = form["action"]
-
-        # Equivalent: Enter token in
-        # https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
-        LOGGER.debug("Attempt to log in to Shibboleth with TFA token")
-        url = "https://idp.scc.kit.edu" + action
-        data = {
-            "_eventId_proceed": "",
-            "j_tokenNumber": self._tfa_auth.get_token()
-        }
-        return soupify(session.post(url, data=data))
-
-    @staticmethod
-    def _login_successful(soup: bs4.BeautifulSoup) -> bool:
-        relay_state = soup.find("input", {"name": "RelayState"})
-        saml_response = soup.find("input", {"name": "SAMLResponse"})
-        return relay_state is not None and saml_response is not None
-
-    @staticmethod
-    def _tfa_required(soup: bs4.BeautifulSoup) -> bool:
-        return soup.find(id="j_tokenNumber") is not None

PFERD/ilias/crawler.py

@@ -1,644 +0,0 @@
-"""
-Contains an ILIAS crawler alongside helper functions.
-"""
-
-import datetime
-import json
-import logging
-import re
-from enum import Enum
-from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Union
-from urllib.parse import (parse_qs, urlencode, urljoin, urlparse, urlsplit,
-                          urlunsplit)
-
-import bs4
-import requests
-
-from ..errors import FatalException
-from ..logging import PrettyLogger
-from ..utils import soupify
-from .authenticators import IliasAuthenticator
-from .date_demangler import demangle_date
-from .downloader import IliasDownloadInfo
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-class IliasElementType(Enum):
-    """
-    The type of an ilias element.
-    """
-    REGULAR_FOLDER = "REGULAR_FOLDER"
-    VIDEO_FOLDER = "VIDEO_FOLDER"
-    EXERCISE_FOLDER = "EXERCISE_FOLDER"
-    REGULAR_FILE = "REGULAR_FILE"
-    VIDEO_FILE = "VIDEO_FILE"
-    FORUM = "FORUM"
-    EXTERNAL_LINK = "EXTERNAL_LINK"
-
-    def is_folder(self) -> bool:
-        """
-        Returns whether this type is some kind of folder.
-        """
-        return "FOLDER" in str(self.name)
-
-
-IliasDirectoryFilter = Callable[[Path, IliasElementType], bool]
-
-
-class IliasCrawlerEntry:
-    # pylint: disable=too-few-public-methods
-    """
-    An ILIAS crawler entry used internally to find, catalogue and recursively crawl elements.
-    """
-
-    def __init__(
-            self,
-            path: Path,
-            url: Union[str, Callable[[], Optional[str]]],
-            entry_type: IliasElementType,
-            modification_date: Optional[datetime.datetime]
-    ):
-        self.path = path
-        if isinstance(url, str):
-            str_url = url
-            self.url: Callable[[], Optional[str]] = lambda: str_url
-        else:
-            self.url = url
-        self.entry_type = entry_type
-        self.modification_date = modification_date
-
-    def to_download_info(self) -> Optional[IliasDownloadInfo]:
-        """
-        Converts this crawler entry to an IliasDownloadInfo, if possible.
-        This method will only succeed for *File* types.
-        """
-        if self.entry_type in [IliasElementType.REGULAR_FILE, IliasElementType.VIDEO_FILE]:
-            return IliasDownloadInfo(self.path, self.url, self.modification_date)
-        return None
-
-
-class IliasCrawler:
-    # pylint: disable=too-few-public-methods
-
-    """
-    A crawler for ILIAS.
-    """
-
-    # pylint: disable=too-many-arguments
-    def __init__(
-            self,
-            base_url: str,
-            session: requests.Session,
-            authenticator: IliasAuthenticator,
-            dir_filter: IliasDirectoryFilter
-    ):
-        """
-        Create a new ILIAS crawler.
-        """
-
-        self._base_url = base_url
-        self._session = session
-        self._authenticator = authenticator
-        self.dir_filter = dir_filter
-
-    @staticmethod
-    def _url_set_query_param(url: str, param: str, value: str) -> str:
-        """
-        Set a query parameter in an url, overwriting existing ones with the same name.
-        """
-        scheme, netloc, path, query, fragment = urlsplit(url)
-        query_parameters = parse_qs(query)
-        query_parameters[param] = [value]
-        new_query_string = urlencode(query_parameters, doseq=True)
-
-        return urlunsplit((scheme, netloc, path, new_query_string, fragment))
-
-    def recursive_crawl_url(self, url: str) -> List[IliasDownloadInfo]:
-        """
-        Crawls a given url *and all reachable elements in it*.
-
-        Args:
-            url {str} -- the *full* url to crawl
-        """
-        start_entries: List[IliasCrawlerEntry] = self._crawl_folder(Path(""), url)
-        return self._iterate_entries_to_download_infos(start_entries)
-
-    def crawl_course(self, course_id: str) -> List[IliasDownloadInfo]:
-        """
-        Starts the crawl process for a course, yielding a list of elements to (potentially)
-        download.
-
-        Arguments:
-            course_id {str} -- the course id
-
-        Raises:
-            FatalException: if an unrecoverable error occurs or the course id is not valid
-        """
-        # Start crawling at the given course
-        root_url = self._url_set_query_param(
-            self._base_url + "/goto.php", "target", f"crs_{course_id}"
-        )
-
-        if not self._is_course_id_valid(root_url, course_id):
-            raise FatalException(
-                "Invalid course id? I didn't find anything looking like a course!"
-            )
-
-        # And treat it as a folder
-        entries: List[IliasCrawlerEntry] = self._crawl_folder(Path(""), root_url)
-        return self._iterate_entries_to_download_infos(entries)
-
-    def _is_course_id_valid(self, root_url: str, course_id: str) -> bool:
-        response: requests.Response = self._session.get(root_url)
-        # We were redirected ==> Non-existant ID
-        if course_id not in response.url:
-            return False
-
-        link_element: bs4.Tag = self._get_page(root_url, {}).find(id="current_perma_link")
-        if not link_element:
-            return False
-        # It wasn't a course but a category list, forum, etc.
-        return "crs_" in link_element.get("value")
-
-    def find_course_name(self, course_id: str) -> Optional[str]:
-        """
-        Returns the name of a given course. None if it is not a valid course
-        or it could not be found.
-        """
-        course_url = self._url_set_query_param(
-            self._base_url + "/goto.php", "target", f"crs_{course_id}"
-        )
-        return self.find_element_name(course_url)
-
-    def find_element_name(self, url: str) -> Optional[str]:
-        """
-        Returns the name of the element at the given URL, if it can find one.
-        """
-        focus_element: bs4.Tag = self._get_page(url, {}).find(id="il_mhead_t_focus")
-        if not focus_element:
-            return None
-        return focus_element.text
-
-    def crawl_personal_desktop(self) -> List[IliasDownloadInfo]:
-        """
-        Crawls the ILIAS personal desktop (and every subelements that can be reached from there).
-
-        Raises:
-            FatalException: if an unrecoverable error occurs
-        """
-        entries: List[IliasCrawlerEntry] = self._crawl_folder(
-            Path(""), self._base_url + "?baseClass=ilPersonalDesktopGUI"
-        )
-        return self._iterate_entries_to_download_infos(entries)
-
-    def _iterate_entries_to_download_infos(
-            self,
-            entries: List[IliasCrawlerEntry]
-    ) -> List[IliasDownloadInfo]:
-        result: List[IliasDownloadInfo] = []
-        entries_to_process: List[IliasCrawlerEntry] = entries.copy()
-        while len(entries_to_process) > 0:
-            entry = entries_to_process.pop()
-
-            if entry.entry_type == IliasElementType.EXTERNAL_LINK:
-                PRETTY.not_searching(entry.path, "external link")
-                continue
-            if entry.entry_type == IliasElementType.FORUM:
-                PRETTY.not_searching(entry.path, "forum")
-                continue
-
-            if entry.entry_type.is_folder() and not self.dir_filter(entry.path, entry.entry_type):
-                PRETTY.not_searching(entry.path, "user filter")
-                continue
-
-            download_info = entry.to_download_info()
-            if download_info is not None:
-                result.append(download_info)
-                continue
-
-            url = entry.url()
-
-            if url is None:
-                PRETTY.warning(f"Could not find url for {str(entry.path)!r}, skipping it")
-                continue
-
-            PRETTY.searching(entry.path)
-
-            if entry.entry_type == IliasElementType.EXERCISE_FOLDER:
-                entries_to_process += self._crawl_exercises(entry.path, url)
-                continue
-            if entry.entry_type == IliasElementType.REGULAR_FOLDER:
-                entries_to_process += self._crawl_folder(entry.path, url)
-                continue
-            if entry.entry_type == IliasElementType.VIDEO_FOLDER:
-                entries_to_process += self._crawl_video_directory(entry.path, url)
-                continue
-
-        return result
-
-    def _crawl_folder(self, folder_path: Path, url: str) -> List[IliasCrawlerEntry]:
-        """
-        Crawl all files in a folder-like element.
-        """
-        soup = self._get_page(url, {})
-
-        if soup.find(id="headerimage"):
-            element: bs4.Tag = soup.find(id="headerimage")
-            if "opencast" in element.attrs["src"].lower():
-                PRETTY.warning(f"Switched to crawling a video at {folder_path}")
-                if not self.dir_filter(folder_path, IliasElementType.VIDEO_FOLDER):
-                    PRETTY.not_searching(folder_path, "user filter")
-                    return []
-                return self._crawl_video_directory(folder_path, url)
-
-        result: List[IliasCrawlerEntry] = []
-
-        # Fetch all links and throw them to the general interpreter
-        links: List[bs4.Tag] = soup.select("a.il_ContainerItemTitle")
-        for link in links:
-            abs_url = self._abs_url_from_link(link)
-            element_path = Path(folder_path, link.getText().strip())
-            element_type = self._find_type_from_link(element_path, link, abs_url)
-
-            if element_type == IliasElementType.REGULAR_FILE:
-                result += self._crawl_file(folder_path, link, abs_url)
-            elif element_type is not None:
-                result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
-            else:
-                PRETTY.warning(f"Found element without a type at {str(element_path)!r}")
-
-        return result
-
-    def _abs_url_from_link(self, link_tag: bs4.Tag) -> str:
-        """
-        Create an absolute url from an <a> tag.
-        """
-        return urljoin(self._base_url, link_tag.get("href"))
-
-    @staticmethod
-    def _find_type_from_link(
-            path: Path,
-            link_element: bs4.Tag,
-            url: str
-    ) -> Optional[IliasElementType]:
-        """
-        Decides which sub crawler to use for a given top level element.
-        """
-        parsed_url = urlparse(url)
-        LOGGER.debug("Parsed url: %r", parsed_url)
-
-        # file URLs contain "target=file"
-        if "target=file_" in parsed_url.query:
-            return IliasElementType.REGULAR_FILE
-
-        # Skip forums
-        if "cmd=showThreads" in parsed_url.query:
-            return IliasElementType.FORUM
-
-        # Everything with a ref_id can *probably* be opened to reveal nested things
-        # video groups, directories, exercises, etc
-        if "ref_id=" in parsed_url.query:
-            return IliasCrawler._find_type_from_folder_like(link_element, url)
-
-        PRETTY.warning(
-            "Got unknown element type in switch. I am not sure what horror I found on the"
-            f" ILIAS page. The element was at {str(path)!r} and it is {link_element!r})"
-        )
-        return None
-
-    @staticmethod
-    def _find_type_from_folder_like(link_element: bs4.Tag, url: str) -> Optional[IliasElementType]:
-        """
-        Try crawling something that looks like a folder.
-        """
-        # pylint: disable=too-many-return-statements
-
-        # We look for the outer div of our inner link, to find information around it
-        # (mostly the icon)
-        for parent in link_element.parents:
-            if "ilContainerListItemOuter" in parent["class"]:
-                found_parent = parent
-                break
-
-        if found_parent is None:
-            PRETTY.warning(f"Could not find element icon for {url!r}")
-            return None
-
-        # Find the small descriptive icon to figure out the type
-        img_tag: Optional[bs4.Tag] = found_parent.select_one("img.ilListItemIcon")
-
-        if img_tag is None:
-            PRETTY.warning(f"Could not find image tag for {url!r}")
-            return None
-
-        if "opencast" in str(img_tag["alt"]).lower():
-            return IliasElementType.VIDEO_FOLDER
-
-        if str(img_tag["src"]).endswith("icon_exc.svg"):
-            return IliasElementType.EXERCISE_FOLDER
-
-        if str(img_tag["src"]).endswith("icon_webr.svg"):
-            return IliasElementType.EXTERNAL_LINK
-
-        if str(img_tag["src"]).endswith("frm.svg"):
-            return IliasElementType.FORUM
-
-        return IliasElementType.REGULAR_FOLDER
-
-    @staticmethod
-    def _crawl_file(path: Path, link_element: bs4.Tag, url: str) -> List[IliasCrawlerEntry]:
-        """
-        Crawls a file.
-        """
-        # Files have a list of properties (type, modification date, size, etc.)
-        # In a series of divs.
-        # Find the parent containing all those divs, so we can filter our what we need
-        properties_parent: bs4.Tag = link_element.findParent(
-            "div", {"class": lambda x: "il_ContainerListItem" in x}
-        ).select_one(".il_ItemProperties")
-        # The first one is always the filetype
-        file_type = properties_parent.select_one("span.il_ItemProperty").getText().strip()
-
-        # The rest does not have a stable order. Grab the whole text and reg-ex the date
-        # out of it
-        all_properties_text = properties_parent.getText().strip()
-        modification_date_match = re.search(
-            r"(((\d+\. \w+ \d+)|(Gestern|Yesterday)|(Heute|Today)|(Morgen|Tomorrow)), \d+:\d+)",
-            all_properties_text
-        )
-        if modification_date_match is None:
-            modification_date = None
-            PRETTY.warning(f"Could not extract start date from {all_properties_text!r}")
-        else:
-            modification_date_str = modification_date_match.group(1)
-            modification_date = demangle_date(modification_date_str)
-
-        # Grab the name from the link text
-        name = link_element.getText()
-        full_path = Path(path, name + "." + file_type)
-
-        return [
-            IliasCrawlerEntry(full_path, url, IliasElementType.REGULAR_FILE, modification_date)
-        ]
-
-    def _crawl_video_directory(self, video_dir_path: Path, url: str) -> List[IliasCrawlerEntry]:
-        """
-        Crawl the video overview site.
-        """
-        initial_soup = self._get_page(url, {})
-
-        # The page is actually emtpy but contains a much needed token in the link below.
-        # That token can be used to fetch the *actual* video listing
-        content_link: bs4.Tag = initial_soup.select_one("#tab_series a")
-        # Fetch the actual video listing. The given parameters return all videos (max 800)
-        # in a standalone html page
-        video_list_soup = self._get_page(
-            self._abs_url_from_link(content_link),
-            {"limit": 800, "cmd": "asyncGetTableGUI", "cmdMode": "asynch"}
-        )
-
-        # If we find a page selected, we probably need to respect pagination
-        if self._is_paginated_video_page(video_list_soup):
-            second_stage_url = self._abs_url_from_link(content_link)
-
-            return self._crawl_paginated_video_directory(
-                video_dir_path, video_list_soup, second_stage_url
-            )
-
-        return self._crawl_video_directory_second_stage(video_dir_path, video_list_soup)
-
-    @staticmethod
-    def _is_paginated_video_page(soup: bs4.BeautifulSoup) -> bool:
-        return soup.find(id=re.compile(r"tab_page_sel.+")) is not None
-
-    def _crawl_paginated_video_directory(
-            self,
-            video_dir_path: Path,
-            paged_video_list_soup: bs4.BeautifulSoup,
-            second_stage_url: str
-    ) -> List[IliasCrawlerEntry]:
-        LOGGER.info("Found paginated video page, trying 800 elements")
-
-        # Try to find the table id. This can be used to build the query parameter indicating
-        # you want 800 elements
-
-        table_element: bs4.Tag = paged_video_list_soup.find(
-            name="table", id=re.compile(r"tbl_xoct_.+")
-        )
-        if table_element is None:
-            PRETTY.warning(
-                "Could not increase elements per page (table not found)."
-                " Some might not be crawled!"
-            )
-            return self._crawl_video_directory_second_stage(video_dir_path, paged_video_list_soup)
-
-        match = re.match(r"tbl_xoct_(.+)", table_element.attrs["id"])
-        if match is None:
-            PRETTY.warning(
-                "Could not increase elements per page (table id not found)."
-                " Some might not be crawled!"
-            )
-            return self._crawl_video_directory_second_stage(video_dir_path, paged_video_list_soup)
-        table_id = match.group(1)
-
-        extended_video_page = self._get_page(
-            second_stage_url,
-            {f"tbl_xoct_{table_id}_trows": 800, "cmd": "asyncGetTableGUI", "cmdMode": "asynch"}
-        )
-
-        if self._is_paginated_video_page(extended_video_page):
-            PRETTY.warning(
-                "800 elements do not seem to be enough (or I failed to fetch that many)."
-                " I will miss elements."
-            )
-
-        return self._crawl_video_directory_second_stage(video_dir_path, extended_video_page)
-
-    def _crawl_video_directory_second_stage(
-            self,
-            video_dir_path: Path,
-            video_list_soup: bs4.BeautifulSoup
-    ) -> List[IliasCrawlerEntry]:
-        """
-        Crawls the "second stage" video page. This page contains the actual video urls.
-        """
-        direct_download_links: List[bs4.Tag] = video_list_soup.findAll(
-            name="a", text=re.compile(r"\s*Download\s*")
-        )
-
-        # Video start links are marked with an "Abspielen" link
-        video_links: List[bs4.Tag] = video_list_soup.findAll(
-            name="a", text=re.compile(r"\s*Abspielen\s*")
-        )
-
-        results: List[IliasCrawlerEntry] = []
-
-        # We can download everything directly!
-        # FIXME: Sadly the download button is currently broken, so never do that
-        if False and len(direct_download_links) == len(video_links):
-            for link in direct_download_links:
-                results += self._crawl_single_video(video_dir_path, link, True)
-        else:
-            for link in video_links:
-                results += self._crawl_single_video(video_dir_path, link, False)
-
-        return results
-
-    def _crawl_single_video(
-            self,
-            parent_path: Path,
-            link: bs4.Tag,
-            direct_download: bool
-    ) -> List[IliasCrawlerEntry]:
-        """
-        Crawl a single video based on its "Abspielen" link from the video listing.
-        """
-        # The link is part of a table with multiple columns, describing metadata.
-        # 6th child (1 indexed) is the modification time string
-        modification_string = link.parent.parent.parent.select_one(
-            "td.std:nth-child(6)"
-        ).getText().strip()
-        modification_time = datetime.datetime.strptime(modification_string, "%d.%m.%Y - %H:%M")
-
-        title = link.parent.parent.parent.select_one(
-            "td.std:nth-child(3)"
-        ).getText().strip()
-        title += ".mp4"
-
-        video_path: Path = Path(parent_path, title)
-
-        video_url = self._abs_url_from_link(link)
-
-        # The video had a direct download button we can use instead
-        if direct_download:
-            LOGGER.debug("Using direct download for video %r", str(video_path))
-            return [IliasCrawlerEntry(
-                video_path, video_url, IliasElementType.VIDEO_FILE, modification_time
-            )]
-
-        return [IliasCrawlerEntry(
-            video_path,
-            self._crawl_video_url_from_play_link(video_url),
-            IliasElementType.VIDEO_FILE,
-            modification_time
-        )]
-
-    def _crawl_video_url_from_play_link(self, play_url: str) -> Callable[[], Optional[str]]:
-        def inner() -> Optional[str]:
-            # Fetch the actual video page. This is a small wrapper page initializing a javscript
-            # player. Sadly we can not execute that JS. The actual video stream url is nowhere
-            # on the page, but defined in a JS object inside a script tag, passed to the player
-            # library.
-            # We do the impossible and RegEx the stream JSON object out of the page's HTML source
-            video_page_soup = soupify(self._session.get(play_url))
-            regex: re.Pattern = re.compile(
-                r"({\"streams\"[\s\S]+?),\s*{\"paella_config_file", re.IGNORECASE
-            )
-            json_match = regex.search(str(video_page_soup))
-
-            if json_match is None:
-                PRETTY.warning(f"Could not find json stream info for {play_url!r}")
-                return None
-            json_str = json_match.group(1)
-
-            # parse it
-            json_object = json.loads(json_str)
-            # and fetch the video url!
-            video_url = json_object["streams"][0]["sources"]["mp4"][0]["src"]
-            return video_url
-        return inner
-
-    def _crawl_exercises(self, element_path: Path, url: str) -> List[IliasCrawlerEntry]:
-        """
-        Crawl files offered for download in exercises.
-        """
-        soup = self._get_page(url, {})
-
-        results: List[IliasCrawlerEntry] = []
-
-        # Each assignment is in an accordion container
-        assignment_containers: List[bs4.Tag] = soup.select(".il_VAccordionInnerContainer")
-
-        for container in assignment_containers:
-            # Fetch the container name out of the header to use it in the path
-            container_name = container.select_one(".ilAssignmentHeader").getText().strip()
-            # Find all download links in the container (this will contain all the files)
-            files: List[bs4.Tag] = container.findAll(
-                name="a",
-                # download links contain the given command class
-                attrs={"href": lambda x: x and "cmdClass=ilexsubmissiongui" in x},
-                text="Download"
-            )
-
-            LOGGER.debug("Found exercise container %r", container_name)
-
-            # Grab each file as you now have the link
-            for file_link in files:
-                # Two divs, side by side. Left is the name, right is the link ==> get left
-                # sibling
-                file_name = file_link.parent.findPrevious(name="div").getText().strip()
-                url = self._abs_url_from_link(file_link)
-
-                LOGGER.debug("Found file %r at %r", file_name, url)
-
-                results.append(IliasCrawlerEntry(
-                    Path(element_path, container_name, file_name),
-                    url,
-                    IliasElementType.REGULAR_FILE,
-                    None  # We do not have any timestamp
-                ))
-
-        return results
-
-    def _get_page(self, url: str, params: Dict[str, Any]) -> bs4.BeautifulSoup:
-        """
-        Fetches a page from ILIAS, authenticating when needed.
-        """
-        LOGGER.debug("Fetching %r", url)
-
-        response = self._session.get(url, params=params)
-        content_type = response.headers["content-type"]
-
-        if not content_type.startswith("text/html"):
-            raise FatalException(
-                f"Invalid content type {content_type} when crawling ilias page"
-                " {url!r} with {params!r}"
-            )
-
-        soup = soupify(response)
-
-        if self._is_logged_in(soup):
-            return soup
-
-        LOGGER.info("Not authenticated, changing that...")
-
-        self._authenticator.authenticate(self._session)
-
-        return self._get_page(url, params)
-
-    @staticmethod
-    def _is_logged_in(soup: bs4.BeautifulSoup) -> bool:
-        # Normal ILIAS pages
-        userlog = soup.find("li", {"id": "userlog"})
-        if userlog is not None:
-            LOGGER.debug("Auth: Found #userlog")
-            return True
-        # Video listing embeds do not have complete ILIAS html. Try to match them by
-        # their video listing table
-        video_table = soup.find(
-            recursive=True,
-            name="table",
-            attrs={"id": lambda x: x is not None and x.startswith("tbl_xoct")}
-        )
-        if video_table is not None:
-            LOGGER.debug("Auth: Found #tbl_xoct.+")
-            return True
-        # The individual video player wrapper page has nothing of the above.
-        # Match it by its playerContainer.
-        if soup.select_one("#playerContainer") is not None:
-            LOGGER.debug("Auth: Found #playerContainer")
-            return True
-        return False

PFERD/ilias/date_demangler.py

@@ -1,51 +0,0 @@
-"""
-Helper methods to demangle an ILIAS date.
-"""
-
-import datetime
-import locale
-import logging
-import re
-from typing import Optional
-
-from ..logging import PrettyLogger
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-def demangle_date(date: str) -> Optional[datetime.datetime]:
-    """
-    Demangle a given date in one of the following formats:
-    "Gestern, HH:MM"
-    "Heute, HH:MM"
-    "Morgen, HH:MM"
-    "dd. mon.yyyy, HH:MM
-    """
-    saved = locale.setlocale(locale.LC_ALL)
-    try:
-        try:
-            locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
-        except locale.Error:
-            PRETTY.warning(
-                "Could not set language to german. Assuming you use english everywhere."
-            )
-
-        date = re.sub(r"\s+", " ", date)
-        date = re.sub("Gestern|Yesterday", _yesterday().strftime("%d. %b %Y"), date, re.I)
-        date = re.sub("Heute|Today", datetime.date.today().strftime("%d. %b %Y"), date, re.I)
-        date = re.sub("Morgen|Tomorrow", _tomorrow().strftime("%d. %b %Y"), date, re.I)
-        return datetime.datetime.strptime(date, "%d. %b %Y, %H:%M")
-    except ValueError:
-        PRETTY.warning(f"Could not parse date {date!r}")
-        return None
-    finally:
-        locale.setlocale(locale.LC_ALL, saved)
-
-
-def _yesterday() -> datetime.date:
-    return datetime.date.today() - datetime.timedelta(days=1)
-
-
-def _tomorrow() -> datetime.date:
-    return datetime.date.today() + datetime.timedelta(days=1)

PFERD/ilias/downloader.py

@@ -1,162 +0,0 @@
-"""Contains a downloader for ILIAS."""
-
-import datetime
-import logging
-import math
-import os
-from pathlib import Path, PurePath
-from typing import Callable, List, Optional, Union
-
-import bs4
-import requests
-
-from ..logging import PrettyLogger
-from ..organizer import Organizer
-from ..tmp_dir import TmpDir
-from ..transform import Transformable
-from ..utils import soupify, stream_to_path
-from .authenticators import IliasAuthenticator
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-class ContentTypeException(Exception):
-    """Thrown when the content type of the ilias element can not be handled."""
-
-
-class IliasDownloadInfo(Transformable):
-    """
-    This class describes a single file to be downloaded.
-    """
-
-    def __init__(
-            self,
-            path: PurePath,
-            url: Union[str, Callable[[], Optional[str]]],
-            modifcation_date: Optional[datetime.datetime]
-    ):
-        super().__init__(path)
-        if isinstance(url, str):
-            string_url = url
-            self.url: Callable[[], Optional[str]] = lambda: string_url
-        else:
-            self.url = url
-        self.modification_date = modifcation_date
-
-
-IliasDownloadStrategy = Callable[[Organizer, IliasDownloadInfo], bool]
-
-
-def download_everything(organizer: Organizer, info: IliasDownloadInfo) -> bool:
-    # pylint: disable=unused-argument
-    """
-    Accepts everything.
-    """
-    return True
-
-
-def download_modified_or_new(organizer: Organizer, info: IliasDownloadInfo) -> bool:
-    """
-    Accepts new files or files with a more recent modification date.
-    """
-    resolved_file = organizer.resolve(info.path)
-    if not resolved_file.exists() or info.modification_date is None:
-        return True
-    resolved_mod_time_seconds = resolved_file.stat().st_mtime
-
-    # Download if the info is newer
-    if info.modification_date.timestamp() > resolved_mod_time_seconds:
-        return True
-
-    PRETTY.ignored_file(info.path, "local file has newer or equal modification time")
-    return False
-
-
-class IliasDownloader:
-    # pylint: disable=too-many-arguments
-    """A downloader for ILIAS."""
-
-    def __init__(
-            self,
-            tmp_dir: TmpDir,
-            organizer: Organizer,
-            session: requests.Session,
-            authenticator: IliasAuthenticator,
-            strategy: IliasDownloadStrategy,
-            timeout: int = 5
-    ):
-        """
-        Create a new IliasDownloader.
-
-        The timeout applies to the download request only, as bwcloud uses IPv6
-        and requests has a problem with that: https://github.com/psf/requests/issues/5522
-        """
-
-        self._tmp_dir = tmp_dir
-        self._organizer = organizer
-        self._session = session
-        self._authenticator = authenticator
-        self._strategy = strategy
-        self._timeout = timeout
-
-    def download_all(self, infos: List[IliasDownloadInfo]) -> None:
-        """
-        Download multiple files one after the other.
-        """
-
-        for info in infos:
-            self.download(info)
-
-    def download(self, info: IliasDownloadInfo) -> None:
-        """
-        Download a file from ILIAS.
-
-        Retries authentication until eternity if it could not fetch the file.
-        """
-
-        LOGGER.debug("Downloading %r", info)
-        if not self._strategy(self._organizer, info):
-            self._organizer.mark(info.path)
-            return
-
-        tmp_file = self._tmp_dir.new_path()
-
-        while not self._try_download(info, tmp_file):
-            LOGGER.info("Retrying download: %r", info)
-            self._authenticator.authenticate(self._session)
-
-        dst_path = self._organizer.accept_file(tmp_file, info.path)
-        if dst_path and info.modification_date:
-            os.utime(
-                dst_path,
-                times=(
-                    math.ceil(info.modification_date.timestamp()),
-                    math.ceil(info.modification_date.timestamp())
-                )
-            )
-
-    def _try_download(self, info: IliasDownloadInfo, target: Path) -> bool:
-        url = info.url()
-        if url is None:
-            PRETTY.warning(f"Could not download {str(info.path)!r} as I got no URL :/")
-            return True
-
-        with self._session.get(url, stream=True, timeout=self._timeout) as response:
-            content_type = response.headers["content-type"]
-            has_content_disposition = "content-disposition" in response.headers
-
-            if content_type.startswith("text/html") and not has_content_disposition:
-                if self._is_logged_in(soupify(response)):
-                    raise ContentTypeException("Attempting to download a web page, not a file")
-
-                return False
-
-            # Yay, we got the file :)
-            stream_to_path(response, target, info.path.name)
-            return True
-
-    @staticmethod
-    def _is_logged_in(soup: bs4.BeautifulSoup) -> bool:
-        userlog = soup.find("li", {"id": "userlog"})
-        return userlog is not None

PFERD/limiter.py

@@ -0,0 +1,97 @@
+import asyncio
+import time
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import AsyncIterator, Optional
+
+
+@dataclass
+class Slot:
+    active: bool = False
+    last_left: Optional[float] = None
+
+
+class Limiter:
+    def __init__(
+            self,
+            task_limit: int,
+            download_limit: int,
+            task_delay: float
+    ):
+        if task_limit <= 0:
+            raise ValueError("task limit must be at least 1")
+        if download_limit <= 0:
+            raise ValueError("download limit must be at least 1")
+        if download_limit > task_limit:
+            raise ValueError("download limit can't be greater than task limit")
+        if task_delay < 0:
+            raise ValueError("Task delay must not be negative")
+
+        self._slots = [Slot() for _ in range(task_limit)]
+        self._downloads = download_limit
+        self._delay = task_delay
+
+        self._condition = asyncio.Condition()
+
+    def _acquire_slot(self) -> Optional[Slot]:
+        for slot in self._slots:
+            if not slot.active:
+                slot.active = True
+                return slot
+
+        return None
+
+    async def _wait_for_slot_delay(self, slot: Slot) -> None:
+        if slot.last_left is not None:
+            delay = slot.last_left + self._delay - time.time()
+            if delay > 0:
+                await asyncio.sleep(delay)
+
+    def _release_slot(self, slot: Slot) -> None:
+        slot.last_left = time.time()
+        slot.active = False
+
+    @asynccontextmanager
+    async def limit_crawl(self) -> AsyncIterator[None]:
+        slot: Slot
+        async with self._condition:
+            while True:
+                if found_slot := self._acquire_slot():
+                    slot = found_slot
+                    break
+                await self._condition.wait()
+
+        await self._wait_for_slot_delay(slot)
+
+        try:
+            yield
+        finally:
+            async with self._condition:
+                self._release_slot(slot)
+                self._condition.notify_all()
+
+    @asynccontextmanager
+    async def limit_download(self) -> AsyncIterator[None]:
+        slot: Slot
+        async with self._condition:
+            while True:
+                if self._downloads <= 0:
+                    await self._condition.wait()
+                    continue
+
+                if found_slot := self._acquire_slot():
+                    slot = found_slot
+                    self._downloads -= 1
+                    break
+
+                await self._condition.wait()
+
+        await self._wait_for_slot_delay(slot)
+
+        try:
+            yield
+        finally:
+            async with self._condition:
+                self._release_slot(slot)
+                self._downloads += 1
+                self._condition.notify_all()

PFERD/location.py

@@ -1,41 +0,0 @@
-"""
-Contains a Location class for objects with an inherent path.
-"""
-
-from pathlib import Path, PurePath
-
-
-class ResolveException(Exception):
-    """An exception while resolving a file."""
-    # TODO take care of this when doing exception handling
-
-
-class Location:
-    """
-    An object that has an inherent path.
-    """
-
-    def __init__(self, path: Path):
-        self._path = path.resolve()
-
-    @property
-    def path(self) -> Path:
-        """
-        This object's location.
-        """
-
-        return self._path
-
-    def resolve(self, target: PurePath) -> Path:
-        """
-        Resolve a file relative to the path of this location.
-
-        Raises a [ResolveException] if the file is outside the given directory.
-        """
-        absolute_path = self.path.joinpath(target).resolve()
-
-        # TODO Make this less inefficient
-        if self.path not in absolute_path.parents:
-            raise ResolveException(f"Path {target} is not inside directory {self.path}")
-
-        return absolute_path

PFERD/logging.py

@@ -1,187 +1,291 @@
-"""
-Contains a few logger utility functions and implementations.
-"""
+import asyncio
+import sys
+import traceback
+from contextlib import asynccontextmanager, contextmanager
+# TODO In Python 3.9 and above, ContextManager is deprecated
+from typing import AsyncIterator, ContextManager, Iterator, List, Optional
 
-import logging
-from pathlib import Path
-from typing import List, Optional
-
-from rich import print as rich_print
-from rich._log_render import LogRender
-from rich.console import Console
+from rich.console import Console, Group
+from rich.live import Live
+from rich.markup import escape
 from rich.panel import Panel
-from rich.style import Style
-from rich.text import Text
-from rich.theme import Theme
-
-from .download_summary import DownloadSummary
-from .utils import PathLike, to_path
-
-STYLE = "{"
-FORMAT = "[{levelname:<7}] {message}"
-DATE_FORMAT = "%F %T"
+from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID, TextColumn, TimeRemainingColumn,
+                           TransferSpeedColumn)
+from rich.table import Column
 
 
-def enable_logging(name: str = "PFERD", level: int = logging.INFO) -> None:
-    """
-    Enable and configure logging via the logging module.
-    """
+class ProgressBar:
+    def __init__(self, progress: Progress, taskid: TaskID):
+        self._progress = progress
+        self._taskid = taskid
 
-    logger = logging.getLogger(name)
-    logger.setLevel(level)
-    logger.addHandler(RichLoggingHandler(level=level))
+    def advance(self, amount: float = 1) -> None:
+        self._progress.advance(self._taskid, advance=amount)
 
-    # This should be logged by our own handler, and not the root logger's
-    # default handler, so we don't pass it on to the root logger.
-    logger.propagate = False
+    def set_total(self, total: float) -> None:
+        self._progress.update(self._taskid, total=total)
+        self._progress.start_task(self._taskid)
 
 
-class RichLoggingHandler(logging.Handler):
-    """
-    A logging handler that uses rich for highlighting
-    """
+class Log:
+    STATUS_WIDTH = 11
 
-    def __init__(self, level: int) -> None:
-        super().__init__(level=level)
-        self.console = Console(theme=Theme({
-            "logging.level.warning": Style(color="yellow")
-        }))
-        self._log_render = LogRender(show_level=True, show_time=False, show_path=False)
+    def __init__(self) -> None:
+        self.console = Console(highlight=False)
 
-    def emit(self, record: logging.LogRecord) -> None:
-        """
-        Invoked by logging.
-        """
-        log_style = f"logging.level.{record.levelname.lower()}"
-        message = self.format(record)
-
-        level = Text()
-        level.append(record.levelname, log_style)
-        message_text = Text.from_markup(message)
-
-        self.console.print(
-            self._log_render(
-                self.console,
-                [message_text],
-                level=level,
-            )
+        self._crawl_progress = Progress(
+            TextColumn("{task.description}", table_column=Column(ratio=1)),
+            BarColumn(),
+            TimeRemainingColumn(),
+            expand=True,
+        )
+        self._download_progress = Progress(
+            TextColumn("{task.description}", table_column=Column(ratio=1)),
+            TransferSpeedColumn(),
+            DownloadColumn(),
+            BarColumn(),
+            TimeRemainingColumn(),
+            expand=True,
         )
 
+        self._live = Live(console=self.console, transient=True)
+        self._update_live()
 
-class PrettyLogger:
-    """
-    A logger that prints some specially formatted log messages in color.
-    """
+        self._showing_progress = False
+        self._progress_suspended = False
+        self._lock = asyncio.Lock()
+        self._lines: List[str] = []
 
-    def __init__(self, logger: logging.Logger) -> None:
-        self.logger = logger
+        # Whether different parts of the output are enabled or disabled
+        self.output_explain = False
+        self.output_status = True
+        self.output_not_deleted = True
+        self.output_report = True
 
-    @staticmethod
-    def _format_path(path: PathLike) -> str:
-        return repr(str(to_path(path)))
+    def _update_live(self) -> None:
+        elements = []
+        if self._crawl_progress.task_ids:
+            elements.append(self._crawl_progress)
+        if self._download_progress.task_ids:
+            elements.append(self._download_progress)
 
-    def error(self, message: str) -> None:
+        group = Group(*elements)
+        self._live.update(group)
+
+    @contextmanager
+    def show_progress(self) -> Iterator[None]:
+        if self._showing_progress:
+            raise RuntimeError("Calling 'show_progress' while already showing progress")
+
+        self._showing_progress = True
+        try:
+            with self._live:
+                yield
+        finally:
+            self._showing_progress = False
+
+    @asynccontextmanager
+    async def exclusive_output(self) -> AsyncIterator[None]:
+        if not self._showing_progress:
+            raise RuntimeError("Calling 'exclusive_output' while not showing progress")
+
+        async with self._lock:
+            self._progress_suspended = True
+            self._live.stop()
+            try:
+                yield
+            finally:
+                self._live.start()
+                self._progress_suspended = False
+                for line in self._lines:
+                    self.print(line)
+                self._lines = []
+
+    def unlock(self) -> None:
         """
-        Print an error message indicating some operation fatally failed.
-        """
-        self.logger.error(
-            f"[bold red]{message}[/bold red]"
-        )
+        Get rid of an exclusive output state.
 
-    def warning(self, message: str) -> None:
-        """
-        Print a warning message indicating some operation failed, but the error can be recovered
-        or ignored.
-        """
-        self.logger.warning(
-            f"[bold yellow]{message}[/bold yellow]"
-        )
-
-    def modified_file(self, path: PathLike) -> None:
-        """
-        An existing file has changed.
+        This function is meant to let PFERD print log messages after the event
+        loop was forcibly stopped and if it will not be started up again. After
+        this is called, it is not safe to use any functions except the logging
+        functions (print, warn, ...).
         """
 
-        self.logger.info(
-            f"[bold magenta]Modified {self._format_path(path)}.[/bold magenta]"
-        )
+        self._progress_suspended = False
+        for line in self._lines:
+            self.print(line)
 
-    def new_file(self, path: PathLike) -> None:
+    def print(self, text: str) -> None:
         """
-        A new file has been downloaded.
+        Print a normal message. Allows markup.
         """
 
-        self.logger.info(
-            f"[bold green]Created {self._format_path(path)}.[/bold green]"
-        )
+        if self._progress_suspended:
+            self._lines.append(text)
+        else:
+            self.console.print(text)
 
-    def deleted_file(self, path: PathLike) -> None:
+    # TODO Print errors (and warnings?) to stderr
+
+    def warn(self, text: str) -> None:
         """
-        A file has been deleted.
+        Print a warning message. Allows no markup.
         """
 
-        self.logger.info(
-            f"[bold red]Deleted {self._format_path(path)}.[/bold red]"
-        )
+        self.print(f"[bold bright_red]Warning[/] {escape(text)}")
 
-    def ignored_file(self, path: PathLike, reason: str) -> None:
+    def warn_contd(self, text: str) -> None:
         """
-        File was not downloaded or modified.
+        Print further lines of a warning message. Allows no markup.
         """
 
-        self.logger.info(
-            f"[dim]Ignored {self._format_path(path)} "
-            f"([/dim]{reason}[dim]).[/dim]"
-        )
+        self.print(f"{escape(text)}")
 
-    def searching(self, path: PathLike) -> None:
+    def error(self, text: str) -> None:
         """
-        A crawler searches a particular object.
+        Print an error message. Allows no markup.
         """
 
-        self.logger.info(f"Searching {self._format_path(path)}")
+        self.print(f"[bold bright_red]Error[/] [red]{escape(text)}")
 
-    def not_searching(self, path: PathLike, reason: str) -> None:
+    def error_contd(self, text: str) -> None:
         """
-        A crawler does not search a particular object.
+        Print further lines of an error message. Allows no markup.
         """
 
-        self.logger.info(
-            f"[dim]Not searching {self._format_path(path)} "
-            f"([/dim]{reason}[dim]).[/dim]"
-        )
+        self.print(f"[red]{escape(text)}")
 
-    def summary(self, download_summary: DownloadSummary) -> None:
+    def unexpected_exception(self) -> None:
         """
-        Prints a download summary.
+        Call this in an "except" clause to log an unexpected exception.
         """
-        self.logger.info("")
-        self.logger.info("[bold cyan]Download Summary[/bold cyan]")
-        if not download_summary.has_updates():
-            self.logger.info("[bold dim]Nothing changed![/bold dim]")
-            return
 
-        for new_file in download_summary.new_files:
-            self.new_file(new_file)
-        for modified_file in download_summary.modified_files:
-            self.modified_file(modified_file)
-        for deleted_files in download_summary.deleted_files:
-            self.deleted_file(deleted_files)
+        t, v, tb = sys.exc_info()
+        if t is None or v is None or tb is None:
+            # We're not currently handling an exception, so somebody probably
+            # called this function where they shouldn't.
+            self.error("Something unexpected happened")
+            self.error_contd("")
+            for line in traceback.format_stack():
+                self.error_contd(line[:-1])  # Without the newline
+            self.error_contd("")
+        else:
+            self.error("An unexpected exception occurred")
+            self.error_contd("")
+            self.error_contd(traceback.format_exc())
 
-    def starting_synchronizer(
+        # Our print function doesn't take types other than strings, but the
+        # underlying rich.print function does. This call is a special case
+        # anyways, and we're calling it internally, so this should be fine.
+        self.print(Panel.fit("""
+Please copy your program output and send it to the PFERD maintainers, either
+directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
+        """.strip()))  # type: ignore
+
+    def explain_topic(self, text: str) -> None:
+        """
+        Print a top-level explain text. Allows no markup.
+        """
+
+        if self.output_explain:
+            self.print(f"[yellow]{escape(text)}")
+
+    def explain(self, text: str) -> None:
+        """
+        Print an indented explain text. Allows no markup.
+        """
+
+        if self.output_explain:
+            self.print(f"  {escape(text)}")
+
+    def status(self, style: str, action: str, text: str, suffix: str = "") -> None:
+        """
+        Print a status update while crawling. Allows markup in the "style"
+        argument which will be applied to the "action" string.
+        """
+
+        if self.output_status:
+            action = escape(f"{action:<{self.STATUS_WIDTH}}")
+            self.print(f"{style}{action}[/] {escape(text)} {suffix}")
+
+    def not_deleted(self, style: str, action: str, text: str, suffix: str = "") -> None:
+        """
+        Print a message for a local only file that wasn't
+        deleted while crawling. Allows markup in the "style"
+        argument which will be applied to the "action" string.
+        """
+
+        if self.output_status and self.output_not_deleted:
+            action = escape(f"{action:<{self.STATUS_WIDTH}}")
+            self.print(f"{style}{action}[/] {escape(text)} {suffix}")
+
+    def report(self, text: str) -> None:
+        """
+        Print a report after crawling. Allows markup.
+        """
+
+        if self.output_report:
+            self.print(text)
+
+    def report_not_deleted(self, text: str) -> None:
+        """
+        Print a report for a local only file that wasn't deleted after crawling. Allows markup.
+        """
+
+        if self.output_report and self.output_not_deleted:
+            self.print(text)
+
+    @contextmanager
+    def _bar(
             self,
-            target_directory: PathLike,
-            synchronizer_name: str,
-            subject: Optional[str] = None,
-    ) -> None:
+            progress: Progress,
+            description: str,
+            total: Optional[float],
+    ) -> Iterator[ProgressBar]:
+        if total is None:
+            # Indeterminate progress bar
+            taskid = progress.add_task(description, start=False)
+        else:
+            taskid = progress.add_task(description, total=total)
+        self._update_live()
+
+        try:
+            yield ProgressBar(progress, taskid)
+        finally:
+            progress.remove_task(taskid)
+            self._update_live()
+
+    def crawl_bar(
+            self,
+            style: str,
+            action: str,
+            text: str,
+            total: Optional[float] = None,
+    ) -> ContextManager[ProgressBar]:
         """
-        A special message marking that a synchronizer has been started.
+        Allows markup in the "style" argument which will be applied to the
+        "action" string.
         """
 
-        subject_str = f"{subject} " if subject else ""
-        self.logger.info("")
-        self.logger.info((
-            f"[bold cyan]Synchronizing "
-            f"{subject_str}to {self._format_path(target_directory)} "
-            f"using the {synchronizer_name} synchronizer.[/bold cyan]"
-        ))
+        action = escape(f"{action:<{self.STATUS_WIDTH}}")
+        description = f"{style}{action}[/] {text}"
+        return self._bar(self._crawl_progress, description, total)
+
+    def download_bar(
+            self,
+            style: str,
+            action: str,
+            text: str,
+            total: Optional[float] = None,
+    ) -> ContextManager[ProgressBar]:
+        """
+        Allows markup in the "style" argument which will be applied to the
+        "action" string.
+        """
+
+        action = escape(f"{action:<{self.STATUS_WIDTH}}")
+        description = f"{style}{action}[/] {text}"
+        return self._bar(self._download_progress, description, total)
+
+
+log = Log()

PFERD/organizer.py

@@ -1,147 +0,0 @@
-"""A simple helper for managing downloaded files.
-
-A organizer is bound to a single directory.
-"""
-
-import filecmp
-import logging
-import os
-import shutil
-from pathlib import Path, PurePath
-from typing import List, Optional, Set
-
-from .download_summary import DownloadSummary
-from .location import Location
-from .logging import PrettyLogger
-from .utils import prompt_yes_no
-
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
-
-
-class FileAcceptException(Exception):
-    """An exception while accepting a file."""
-
-
-class Organizer(Location):
-    """A helper for managing downloaded files."""
-
-    def __init__(self, path: Path):
-        """Create a new organizer for a given path."""
-        super().__init__(path)
-        self._known_files: Set[Path] = set()
-
-        # Keep the root dir
-        self._known_files.add(path.resolve())
-
-        self.download_summary = DownloadSummary()
-
-    def accept_file(self, src: Path, dst: PurePath) -> Optional[Path]:
-        """
-        Move a file to this organizer and mark it.
-
-        Returns the path the file was moved to, to allow the caller to adjust the metadata.
-        As you might still need to adjust the metadata when the file was identical
-        (e.g. update the timestamp), the path is also returned in this case.
-        In all other cases (ignored, not overwritten, etc.) this method returns None.
-        """
-        # Windows limits the path length to 260 for *some* historical reason
-        # If you want longer paths, you will have to add the "\\?\" prefix in front of
-        # your path...
-        # See:
-        # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
-        if os.name == 'nt':
-            src_absolute = Path("\\\\?\\" + str(src.resolve()))
-            dst_absolute = Path("\\\\?\\" + str(self.resolve(dst)))
-        else:
-            src_absolute = src.resolve()
-            dst_absolute = self.resolve(dst)
-
-        if not src_absolute.exists():
-            raise FileAcceptException("Source file does not exist")
-
-        if not src_absolute.is_file():
-            raise FileAcceptException("Source is a directory")
-
-        LOGGER.debug("Copying %s to %s", src_absolute, dst_absolute)
-
-        if self._is_marked(dst):
-            PRETTY.warning(f"File {str(dst_absolute)!r} was already written!")
-            if not prompt_yes_no(f"Overwrite file?", default=False):
-                PRETTY.ignored_file(dst_absolute, "file was written previously")
-                return None
-
-        # Destination file is directory
-        if dst_absolute.exists() and dst_absolute.is_dir():
-            if prompt_yes_no(f"Overwrite folder {dst_absolute} with file?", default=False):
-                shutil.rmtree(dst_absolute)
-            else:
-                PRETTY.warning(f"Could not add file {str(dst_absolute)!r}")
-                return None
-
-        # Destination file exists
-        if dst_absolute.exists() and dst_absolute.is_file():
-            if filecmp.cmp(str(src_absolute), str(dst_absolute), shallow=False):
-                # Bail out, nothing more to do
-                PRETTY.ignored_file(dst_absolute, "same file contents")
-                self.mark(dst)
-                return dst_absolute
-
-            self.download_summary.add_modified_file(dst_absolute)
-            PRETTY.modified_file(dst_absolute)
-        else:
-            self.download_summary.add_new_file(dst_absolute)
-            PRETTY.new_file(dst_absolute)
-
-        # Create parent dir if needed
-        dst_parent_dir: Path = dst_absolute.parent
-        dst_parent_dir.mkdir(exist_ok=True, parents=True)
-
-        # Move file
-        shutil.move(str(src_absolute), str(dst_absolute))
-
-        self.mark(dst)
-
-        return dst_absolute
-
-    def mark(self, path: PurePath) -> None:
-        """Mark a file as used so it will not get cleaned up."""
-        absolute_path = self.resolve(path)
-        self._known_files.add(absolute_path)
-        LOGGER.debug("Tracked %s", absolute_path)
-
-    def _is_marked(self, path: PurePath) -> bool:
-        """
-        Checks whether a file is marked.
-        """
-        absolute_path = self.resolve(path)
-        return absolute_path in self._known_files
-
-    def cleanup(self) -> None:
-        """Remove all untracked files in the organizer's dir."""
-        LOGGER.debug("Deleting all untracked files...")
-
-        self._cleanup(self.path)
-
-    def _cleanup(self, start_dir: Path) -> None:
-        paths: List[Path] = list(start_dir.iterdir())
-
-        # Recursively clean paths
-        for path in paths:
-            if path.is_dir():
-                self._cleanup(path)
-            else:
-                if path.resolve() not in self._known_files:
-                    self._delete_file_if_confirmed(path)
-
-        # Delete dir if it was empty and untracked
-        dir_empty = len(list(start_dir.iterdir())) == 0
-        if start_dir.resolve() not in self._known_files and dir_empty:
-            start_dir.rmdir()
-
-    def _delete_file_if_confirmed(self, path: Path) -> None:
-        prompt = f"Do you want to delete {path}"
-
-        if prompt_yes_no(prompt, False):
-            self.download_summary.add_deleted_file(path)
-            path.unlink()

PFERD/output_dir.py

@@ -0,0 +1,518 @@
+import filecmp
+import json
+import os
+import random
+import shutil
+import string
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from pathlib import Path, PurePath
+from typing import BinaryIO, Iterator, Optional, Tuple
+
+from .logging import log
+from .report import Report, ReportLoadError
+from .utils import ReusableAsyncContextManager, fmt_path, fmt_real_path, prompt_yes_no
+
+SUFFIX_CHARS = string.ascii_lowercase + string.digits
+SUFFIX_LENGTH = 6
+TRIES = 5
+
+
+class OutputDirError(Exception):
+    pass
+
+
+class Redownload(Enum):
+    NEVER = "never"
+    NEVER_SMART = "never-smart"
+    ALWAYS = "always"
+    ALWAYS_SMART = "always-smart"
+
+    @staticmethod
+    def from_string(string: str) -> "Redownload":
+        try:
+            return Redownload(string)
+        except ValueError:
+            raise ValueError("must be one of 'never', 'never-smart',"
+                             " 'always', 'always-smart'")
+
+
+class OnConflict(Enum):
+    PROMPT = "prompt"
+    LOCAL_FIRST = "local-first"
+    REMOTE_FIRST = "remote-first"
+    NO_DELETE = "no-delete"
+    NO_DELETE_PROMPT_OVERWRITE = "no-delete-prompt-overwrite"
+
+    @staticmethod
+    def from_string(string: str) -> "OnConflict":
+        try:
+            return OnConflict(string)
+        except ValueError:
+            raise ValueError("must be one of 'prompt', 'local-first',"
+                             " 'remote-first', 'no-delete', 'no-delete-prompt-overwrite'")
+
+
+@dataclass
+class Heuristics:
+    mtime: Optional[datetime]
+
+
+class FileSink:
+    def __init__(self, file: BinaryIO):
+        self._file = file
+        self._done = False
+
+    @property
+    def file(self) -> BinaryIO:
+        return self._file
+
+    def done(self) -> None:
+        self._done = True
+
+    def is_done(self) -> bool:
+        return self._done
+
+
+@dataclass
+class DownloadInfo:
+    remote_path: PurePath
+    path: PurePath
+    local_path: Path
+    tmp_path: Path
+    heuristics: Heuristics
+    on_conflict: OnConflict
+    success: bool = False
+
+
+class FileSinkToken(ReusableAsyncContextManager[FileSink]):
+    # Whenever this class is entered, it creates a new temporary file and
+    # returns a corresponding FileSink.
+    #
+    # When it is exited again, the file is closed and information about the
+    # download handed back to the OutputDirectory.
+
+    def __init__(
+            self,
+            output_dir: "OutputDirectory",
+            remote_path: PurePath,
+            path: PurePath,
+            local_path: Path,
+            heuristics: Heuristics,
+            on_conflict: OnConflict,
+    ):
+        super().__init__()
+
+        self._output_dir = output_dir
+        self._remote_path = remote_path
+        self._path = path
+        self._local_path = local_path
+        self._heuristics = heuristics
+        self._on_conflict = on_conflict
+
+    async def _on_aenter(self) -> FileSink:
+        tmp_path, file = await self._output_dir._create_tmp_file(self._local_path)
+        sink = FileSink(file)
+
+        async def after_download() -> None:
+            await self._output_dir._after_download(DownloadInfo(
+                self._remote_path,
+                self._path,
+                self._local_path,
+                tmp_path,
+                self._heuristics,
+                self._on_conflict,
+                sink.is_done(),
+            ))
+
+        self._stack.push_async_callback(after_download)
+        self._stack.enter_context(file)
+
+        return sink
+
+
+class OutputDirectory:
+    REPORT_FILE = PurePath(".report")
+
+    def __init__(
+            self,
+            root: Path,
+            redownload: Redownload,
+            on_conflict: OnConflict,
+    ):
+        if os.name == "nt":
+            # Windows limits the path length to 260 for some historical reason.
+            # If you want longer paths, you will have to add the "\\?\" prefix
+            # in front of your path. See:
+            # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
+            self._root = Path("\\\\?\\" + str(root.absolute()))
+        else:
+            self._root = root
+
+        self._redownload = redownload
+        self._on_conflict = on_conflict
+
+        self._report_path = self.resolve(self.REPORT_FILE)
+        self._report = Report()
+        self._prev_report: Optional[Report] = None
+
+        self.register_reserved(self.REPORT_FILE)
+
+    @property
+    def report(self) -> Report:
+        return self._report
+
+    @property
+    def prev_report(self) -> Optional[Report]:
+        return self._prev_report
+
+    def prepare(self) -> None:
+        log.explain_topic(f"Creating base directory at {fmt_real_path(self._root)}")
+
+        try:
+            self._root.mkdir(parents=True, exist_ok=True)
+        except OSError:
+            raise OutputDirError("Failed to create base directory")
+
+    def register_reserved(self, path: PurePath) -> None:
+        self._report.mark_reserved(path)
+
+    def resolve(self, path: PurePath) -> Path:
+        """
+        May throw an OutputDirError.
+        """
+
+        if ".." in path.parts:
+            raise OutputDirError(f"Forbidden segment '..' in path {fmt_path(path)}")
+        if "." in path.parts:
+            raise OutputDirError(f"Forbidden segment '.' in path {fmt_path(path)}")
+
+        return self._root / path
+
+    def _should_download(
+            self,
+            local_path: Path,
+            heuristics: Heuristics,
+            redownload: Redownload,
+            on_conflict: OnConflict,
+    ) -> bool:
+        if not local_path.exists():
+            log.explain("No corresponding file present locally")
+            return True
+
+        if on_conflict == OnConflict.LOCAL_FIRST:
+            # Whatever is here, it will never be overwritten, so we don't need
+            # to download the file.
+            log.explain("Conflict resolution is 'local-first' and path exists")
+            return False
+
+        if not local_path.is_file():
+            # We know that there is *something* here that's not a file.
+            log.explain("Non-file (probably a directory) present locally")
+
+            # If on_conflict is LOCAL_FIRST or NO_DELETE, we know that it would
+            # never be overwritten. It also doesn't have any relevant stats to
+            # update. This means that we don't have to download the file
+            # because we'd just always throw it away again.
+            if on_conflict in {OnConflict.LOCAL_FIRST, OnConflict.NO_DELETE}:
+                log.explain(f"Conflict resolution is {on_conflict.value!r}")
+                return False
+
+            return True
+
+        log.explain(f"Redownload policy is {redownload.value}")
+
+        if redownload == Redownload.NEVER:
+            return False
+        elif redownload == Redownload.ALWAYS:
+            return True
+
+        stat = local_path.stat()
+
+        remote_newer = None
+
+        # Python on Windows crashes when faced with timestamps around the unix epoch
+        if heuristics.mtime and (os.name != "nt" or heuristics.mtime.year > 1970):
+            mtime = heuristics.mtime
+            remote_newer = mtime.timestamp() > stat.st_mtime
+            if remote_newer:
+                log.explain("Remote file seems to be newer")
+            else:
+                log.explain("Remote file doesn't seem to be newer")
+
+        if redownload == Redownload.NEVER_SMART:
+            if remote_newer is None:
+                return False
+            else:
+                return remote_newer
+        elif redownload == Redownload.ALWAYS_SMART:
+            if remote_newer is None:
+                return True
+            else:
+                return remote_newer
+
+        # This should never be reached
+        raise ValueError(f"{redownload!r} is not a valid redownload policy")
+
+    # The following conflict resolution functions all return False if the local
+    # file(s) should be kept and True if they should be replaced by the remote
+    # files.
+
+    async def _conflict_lfrf(
+            self,
+            on_conflict: OnConflict,
+            path: PurePath,
+    ) -> bool:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
+            async with log.exclusive_output():
+                prompt = f"Replace {fmt_path(path)} with remote file?"
+                return await prompt_yes_no(prompt, default=False)
+        elif on_conflict == OnConflict.LOCAL_FIRST:
+            return False
+        elif on_conflict == OnConflict.REMOTE_FIRST:
+            return True
+        elif on_conflict == OnConflict.NO_DELETE:
+            return True
+
+        # This should never be reached
+        raise ValueError(f"{on_conflict!r} is not a valid conflict policy")
+
+    async def _conflict_ldrf(
+            self,
+            on_conflict: OnConflict,
+            path: PurePath,
+    ) -> bool:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
+            async with log.exclusive_output():
+                prompt = f"Recursively delete {fmt_path(path)} and replace with remote file?"
+                return await prompt_yes_no(prompt, default=False)
+        elif on_conflict == OnConflict.LOCAL_FIRST:
+            return False
+        elif on_conflict == OnConflict.REMOTE_FIRST:
+            return True
+        elif on_conflict == OnConflict.NO_DELETE:
+            return False
+
+        # This should never be reached
+        raise ValueError(f"{on_conflict!r} is not a valid conflict policy")
+
+    async def _conflict_lfrd(
+            self,
+            on_conflict: OnConflict,
+            path: PurePath,
+            parent: PurePath,
+    ) -> bool:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
+            async with log.exclusive_output():
+                prompt = f"Delete {fmt_path(parent)} so remote file {fmt_path(path)} can be downloaded?"
+                return await prompt_yes_no(prompt, default=False)
+        elif on_conflict == OnConflict.LOCAL_FIRST:
+            return False
+        elif on_conflict == OnConflict.REMOTE_FIRST:
+            return True
+        elif on_conflict == OnConflict.NO_DELETE:
+            return False
+
+        # This should never be reached
+        raise ValueError(f"{on_conflict!r} is not a valid conflict policy")
+
+    async def _conflict_delete_lf(
+            self,
+            on_conflict: OnConflict,
+            path: PurePath,
+    ) -> bool:
+        if on_conflict == OnConflict.PROMPT:
+            async with log.exclusive_output():
+                prompt = f"Delete {fmt_path(path)}?"
+                return await prompt_yes_no(prompt, default=False)
+        elif on_conflict == OnConflict.LOCAL_FIRST:
+            return False
+        elif on_conflict == OnConflict.REMOTE_FIRST:
+            return True
+        elif on_conflict in {OnConflict.NO_DELETE, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
+            return False
+
+        # This should never be reached
+        raise ValueError(f"{on_conflict!r} is not a valid conflict policy")
+
+    def _tmp_path(self, base: Path, suffix_length: int) -> Path:
+        prefix = "" if base.name.startswith(".") else "."
+        suffix = "".join(random.choices(SUFFIX_CHARS, k=suffix_length))
+        name = f"{prefix}{base.name}.tmp.{suffix}"
+        return base.parent / name
+
+    async def _create_tmp_file(
+            self,
+            local_path: Path,
+    ) -> Tuple[Path, BinaryIO]:
+        """
+        May raise an OutputDirError.
+        """
+
+        # Create tmp file
+        for attempt in range(TRIES):
+            suffix_length = SUFFIX_LENGTH + 2 * attempt
+            tmp_path = self._tmp_path(local_path, suffix_length)
+            try:
+                return tmp_path, open(tmp_path, "xb")
+            except FileExistsError:
+                pass  # Try again
+
+        raise OutputDirError("Failed to create temporary file")
+
+    async def download(
+            self,
+            remote_path: PurePath,
+            path: PurePath,
+            mtime: Optional[datetime] = None,
+            redownload: Optional[Redownload] = None,
+            on_conflict: Optional[OnConflict] = None,
+    ) -> Optional[FileSinkToken]:
+        """
+        May throw an OutputDirError, a MarkDuplicateError or a
+        MarkConflictError.
+        """
+
+        heuristics = Heuristics(mtime)
+        redownload = self._redownload if redownload is None else redownload
+        on_conflict = self._on_conflict if on_conflict is None else on_conflict
+        local_path = self.resolve(path)
+
+        self._report.mark(path)
+
+        if not self._should_download(local_path, heuristics, redownload, on_conflict):
+            return None
+
+        # Detect and solve local-dir-remote-file conflict
+        if local_path.is_dir():
+            log.explain("Conflict: There's a directory in place of the local file")
+            if await self._conflict_ldrf(on_conflict, path):
+                log.explain("Result: Delete the obstructing directory")
+                shutil.rmtree(local_path)
+            else:
+                log.explain("Result: Keep the obstructing directory")
+                return None
+
+        # Detect and solve local-file-remote-dir conflict
+        for parent in path.parents:
+            local_parent = self.resolve(parent)
+            if local_parent.exists() and not local_parent.is_dir():
+                log.explain("Conflict: One of the local file's parents is a file")
+                if await self._conflict_lfrd(on_conflict, path, parent):
+                    log.explain("Result: Delete the obstructing file")
+                    local_parent.unlink()
+                    break
+                else:
+                    log.explain("Result: Keep the obstructing file")
+                    return None
+
+        # Ensure parent directory exists
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        return FileSinkToken(self, remote_path, path, local_path, heuristics, on_conflict)
+
+    def _update_metadata(self, info: DownloadInfo) -> None:
+        if mtime := info.heuristics.mtime:
+            mtimestamp = mtime.timestamp()
+            os.utime(info.local_path, times=(mtimestamp, mtimestamp))
+
+    @contextmanager
+    def _ensure_deleted(self, path: Path) -> Iterator[None]:
+        try:
+            yield
+        finally:
+            path.unlink(missing_ok=True)
+
+    async def _after_download(self, info: DownloadInfo) -> None:
+        with self._ensure_deleted(info.tmp_path):
+            log.status("[bold cyan]", "Downloaded", fmt_path(info.remote_path))
+            log.explain_topic(f"Processing downloaded file for {fmt_path(info.path)}")
+
+            changed = False
+
+            if not info.success:
+                log.explain("Download unsuccessful, aborting")
+                return
+
+            # Solve conflicts arising from existing local file
+            if info.local_path.exists():
+                changed = True
+
+                if filecmp.cmp(info.local_path, info.tmp_path):
+                    log.explain("Contents identical with existing file")
+                    log.explain("Updating metadata of existing file")
+                    self._update_metadata(info)
+                    return
+
+                log.explain("Conflict: The local and remote versions differ")
+                if await self._conflict_lfrf(info.on_conflict, info.path):
+                    log.explain("Result: Replacing local with remote version")
+                else:
+                    log.explain("Result: Keeping local version")
+                    return
+
+            info.tmp_path.replace(info.local_path)
+            log.explain("Updating file metadata")
+            self._update_metadata(info)
+
+            if changed:
+                log.status("[bold bright_yellow]", "Changed", fmt_path(info.path))
+                self._report.change_file(info.path)
+            else:
+                log.status("[bold bright_green]", "Added", fmt_path(info.path))
+                self._report.add_file(info.path)
+
+    async def cleanup(self) -> None:
+        await self._cleanup_dir(self._root, PurePath(), delete_self=False)
+
+    async def _cleanup(self, path: Path, pure: PurePath) -> None:
+        if path.is_dir():
+            await self._cleanup_dir(path, pure)
+        elif path.is_file():
+            await self._cleanup_file(path, pure)
+
+    async def _cleanup_dir(self, path: Path, pure: PurePath, delete_self: bool = True) -> None:
+        for child in sorted(path.iterdir()):
+            pure_child = pure / child.name
+            await self._cleanup(child, pure_child)
+
+        if delete_self:
+            try:
+                path.rmdir()
+            except OSError:
+                pass
+
+    async def _cleanup_file(self, path: Path, pure: PurePath) -> None:
+        if self._report.is_marked(pure):
+            return
+
+        if await self._conflict_delete_lf(self._on_conflict, pure):
+            try:
+                path.unlink()
+                log.status("[bold bright_magenta]", "Deleted", fmt_path(pure))
+                self._report.delete_file(pure)
+            except OSError:
+                pass
+        else:
+            log.not_deleted("[bold bright_magenta]", "Not deleted", fmt_path(pure))
+            self._report.not_delete_file(pure)
+
+    def load_prev_report(self) -> None:
+        log.explain_topic(f"Loading previous report from {fmt_real_path(self._report_path)}")
+        try:
+            self._prev_report = Report.load(self._report_path)
+            log.explain("Loaded report successfully")
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError, ReportLoadError) as e:
+            log.explain("Failed to load report")
+            log.explain(str(e))
+
+    def store_report(self) -> None:
+        log.explain_topic(f"Storing report to {fmt_real_path(self._report_path)}")
+        try:
+            self._report.store(self._report_path)
+            log.explain("Stored report successfully")
+        except OSError as e:
+            log.warn(f"Failed to save report to {fmt_real_path(self._report_path)}")
+            log.warn_contd(str(e))

PFERD/pferd.py

@@ -1,355 +1,194 @@
-"""
-Convenience functions for using PFERD.
-"""
-
-import logging
 from pathlib import Path
-from typing import Callable, List, Optional, Union
+from typing import Dict, List, Optional
 
-from .cookie_jar import CookieJar
-from .diva import (DivaDownloader, DivaDownloadStrategy, DivaPlaylistCrawler,
-                   diva_download_new)
-from .download_summary import DownloadSummary
-from .errors import FatalException, swallow_and_print_errors
-from .ilias import (IliasAuthenticator, IliasCrawler, IliasDirectoryFilter,
-                    IliasDownloader, IliasDownloadInfo, IliasDownloadStrategy,
-                    KitShibbolethAuthenticator, download_modified_or_new)
-from .location import Location
-from .logging import PrettyLogger, enable_logging
-from .organizer import Organizer
-from .tmp_dir import TmpDir
-from .transform import TF, Transform, apply_transform
-from .utils import PathLike, to_path
+from rich.markup import escape
 
-# TODO save known-good cookies as soon as possible
+from .auth import AUTHENTICATORS, Authenticator, AuthError, AuthSection
+from .config import Config, ConfigOptionError
+from .crawl import CRAWLERS, Crawler, CrawlError, CrawlerSection, KitIliasWebCrawler
+from .logging import log
+from .utils import fmt_path
 
 
-LOGGER = logging.getLogger(__name__)
-PRETTY = PrettyLogger(LOGGER)
+class PferdLoadError(Exception):
+    pass
 
 
-class Pferd(Location):
-    # pylint: disable=too-many-arguments
-    """
-    The main entrypoint in your Pferd usage: This class combines a number of
-    useful shortcuts for running synchronizers in a single interface.
-    """
+class Pferd:
+    def __init__(self, config: Config, cli_crawlers: Optional[List[str]], cli_skips: Optional[List[str]]):
+        """
+        May throw PferdLoadError.
+        """
 
-    def __init__(
+        self._config = config
+        self._crawlers_to_run = self._find_crawlers_to_run(config, cli_crawlers, cli_skips)
+
+        self._authenticators: Dict[str, Authenticator] = {}
+        self._crawlers: Dict[str, Crawler] = {}
+
+    def _find_config_crawlers(self, config: Config) -> List[str]:
+        crawl_sections = []
+
+        for name, section in config.crawl_sections():
+            if CrawlerSection(section).skip():
+                log.explain(f"Skipping {name!r}")
+            else:
+                crawl_sections.append(name)
+
+        return crawl_sections
+
+    def _find_cli_crawlers(self, config: Config, cli_crawlers: List[str]) -> List[str]:
+        if len(cli_crawlers) != len(set(cli_crawlers)):
+            raise PferdLoadError("Some crawlers were selected multiple times")
+
+        crawl_sections = [name for name, _ in config.crawl_sections()]
+
+        crawlers_to_run = []  # With crawl: prefix
+        unknown_names = []  # Without crawl: prefix
+
+        for name in cli_crawlers:
+            section_name = f"crawl:{name}"
+            if section_name in crawl_sections:
+                log.explain(f"Crawler section named {section_name!r} exists")
+                crawlers_to_run.append(section_name)
+            else:
+                log.explain(f"There's no crawler section named {section_name!r}")
+                unknown_names.append(name)
+
+        if unknown_names:
+            if len(unknown_names) == 1:
+                [name] = unknown_names
+                raise PferdLoadError(f"There is no crawler named {name!r}")
+            else:
+                names_str = ", ".join(repr(name) for name in unknown_names)
+                raise PferdLoadError(f"There are no crawlers named {names_str}")
+
+        return crawlers_to_run
+
+    def _find_crawlers_to_run(
             self,
-            base_dir: Path,
-            tmp_dir: Path = Path(".tmp"),
-            test_run: bool = False
-    ):
-        super().__init__(Path(base_dir))
+            config: Config,
+            cli_crawlers: Optional[List[str]],
+            cli_skips: Optional[List[str]],
+    ) -> List[str]:
+        log.explain_topic("Deciding which crawlers to run")
 
-        self._download_summary = DownloadSummary()
-        self._tmp_dir = TmpDir(self.resolve(tmp_dir))
-        self._test_run = test_run
-
-    @staticmethod
-    def enable_logging() -> None:
-        """
-        Enable and configure logging via the logging module.
-        """
-
-        enable_logging()
-
-    @staticmethod
-    def _print_transformables(transformables: List[TF]) -> None:
-        LOGGER.info("")
-        LOGGER.info("Results of the test run:")
-        for transformable in transformables:
-            LOGGER.info(transformable.path)
-
-    def _ilias(
-            self,
-            target: PathLike,
-            base_url: str,
-            crawl_function: Callable[[IliasCrawler], List[IliasDownloadInfo]],
-            authenticator: IliasAuthenticator,
-            cookies: Optional[PathLike],
-            dir_filter: IliasDirectoryFilter,
-            transform: Transform,
-            download_strategy: IliasDownloadStrategy,
-            timeout: int,
-            clean: bool = True,
-    ) -> Organizer:
-        # pylint: disable=too-many-locals
-        cookie_jar = CookieJar(to_path(cookies) if cookies else None)
-        session = cookie_jar.create_session()
-        tmp_dir = self._tmp_dir.new_subdir()
-        organizer = Organizer(self.resolve(to_path(target)))
-
-        crawler = IliasCrawler(base_url, session, authenticator, dir_filter)
-        downloader = IliasDownloader(tmp_dir, organizer, session,
-                                     authenticator, download_strategy, timeout)
-
-        cookie_jar.load_cookies()
-        info = crawl_function(crawler)
-        cookie_jar.save_cookies()
-
-        transformed = apply_transform(transform, info)
-        if self._test_run:
-            self._print_transformables(transformed)
-            return organizer
-
-        downloader.download_all(transformed)
-        cookie_jar.save_cookies()
-
-        if clean:
-            organizer.cleanup()
-
-        return organizer
-
-    @swallow_and_print_errors
-    def ilias_kit(
-            self,
-            target: PathLike,
-            course_id: str,
-            dir_filter: IliasDirectoryFilter = lambda x, y: True,
-            transform: Transform = lambda x: x,
-            cookies: Optional[PathLike] = None,
-            username: Optional[str] = None,
-            password: Optional[str] = None,
-            download_strategy: IliasDownloadStrategy = download_modified_or_new,
-            clean: bool = True,
-            timeout: int = 5,
-    ) -> Organizer:
-        """
-        Synchronizes a folder with the ILIAS instance of the KIT.
-
-        Arguments:
-            target {Path} -- the target path to write the data to
-            course_id {str} -- the id of the main course page (found in the URL after ref_id
-                when opening the course homepage)
-
-        Keyword Arguments:
-            dir_filter {IliasDirectoryFilter} -- A filter for directories. Will be applied on the
-                crawler level, these directories and all of their content is skipped.
-                (default: {lambdax:True})
-            transform {Transform} -- A transformation function for the output paths. Return None
-                to ignore a file. (default: {lambdax:x})
-            cookies {Optional[Path]} -- The path to store and load cookies from.
-                (default: {None})
-            username {Optional[str]} -- The SCC username. If none is given, it will prompt
-                the user. (default: {None})
-            password {Optional[str]} -- The SCC password. If none is given, it will prompt
-                the user. (default: {None})
-            download_strategy {DownloadStrategy} -- A function to determine which files need to
-                be downloaded. Can save bandwidth and reduce the number of requests.
-                (default: {download_modified_or_new})
-            clean {bool} -- Whether to clean up when the method finishes.
-            timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
-                requests bug.
-        """
-        # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
-        PRETTY.starting_synchronizer(target, "ILIAS", course_id)
-
-        organizer = self._ilias(
-            target=target,
-            base_url="https://ilias.studium.kit.edu/",
-            crawl_function=lambda crawler: crawler.crawl_course(course_id),
-            authenticator=authenticator,
-            cookies=cookies,
-            dir_filter=dir_filter,
-            transform=transform,
-            download_strategy=download_strategy,
-            clean=clean,
-            timeout=timeout
-        )
-
-        self._download_summary.merge(organizer.download_summary)
-
-        return organizer
-
-    def print_summary(self) -> None:
-        """
-        Prints the accumulated download summary.
-        """
-        PRETTY.summary(self._download_summary)
-
-    @swallow_and_print_errors
-    def ilias_kit_personal_desktop(
-            self,
-            target: PathLike,
-            dir_filter: IliasDirectoryFilter = lambda x, y: True,
-            transform: Transform = lambda x: x,
-            cookies: Optional[PathLike] = None,
-            username: Optional[str] = None,
-            password: Optional[str] = None,
-            download_strategy: IliasDownloadStrategy = download_modified_or_new,
-            clean: bool = True,
-            timeout: int = 5,
-    ) -> Organizer:
-        """
-        Synchronizes a folder with the ILIAS instance of the KIT. This method will crawl the ILIAS
-        "personal desktop" instead of a single course.
-
-        Arguments:
-            target {Path} -- the target path to write the data to
-
-        Keyword Arguments:
-            dir_filter {IliasDirectoryFilter} -- A filter for directories. Will be applied on the
-                crawler level, these directories and all of their content is skipped.
-                (default: {lambdax:True})
-            transform {Transform} -- A transformation function for the output paths. Return None
-                to ignore a file. (default: {lambdax:x})
-            cookies {Optional[Path]} -- The path to store and load cookies from.
-                (default: {None})
-            username {Optional[str]} -- The SCC username. If none is given, it will prompt
-                the user. (default: {None})
-            password {Optional[str]} -- The SCC password. If none is given, it will prompt
-                the user. (default: {None})
-            download_strategy {DownloadStrategy} -- A function to determine which files need to
-                be downloaded. Can save bandwidth and reduce the number of requests.
-                (default: {download_modified_or_new})
-            clean {bool} -- Whether to clean up when the method finishes.
-            timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
-                requests bug.
-        """
-        # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
-        PRETTY.starting_synchronizer(target, "ILIAS", "Personal Desktop")
-
-        organizer = self._ilias(
-            target=target,
-            base_url="https://ilias.studium.kit.edu/",
-            crawl_function=lambda crawler: crawler.crawl_personal_desktop(),
-            authenticator=authenticator,
-            cookies=cookies,
-            dir_filter=dir_filter,
-            transform=transform,
-            download_strategy=download_strategy,
-            clean=clean,
-            timeout=timeout
-        )
-
-        self._download_summary.merge(organizer.download_summary)
-
-        return organizer
-
-    @swallow_and_print_errors
-    def ilias_kit_folder(
-            self,
-            target: PathLike,
-            full_url: str,
-            dir_filter: IliasDirectoryFilter = lambda x, y: True,
-            transform: Transform = lambda x: x,
-            cookies: Optional[PathLike] = None,
-            username: Optional[str] = None,
-            password: Optional[str] = None,
-            download_strategy: IliasDownloadStrategy = download_modified_or_new,
-            clean: bool = True,
-            timeout: int = 5,
-    ) -> Organizer:
-        """
-        Synchronizes a folder with a given folder on the ILIAS instance of the KIT.
-
-        Arguments:
-            target {Path}  -- the target path to write the data to
-            full_url {str} -- the full url of the folder/videos/course to crawl
-
-        Keyword Arguments:
-            dir_filter {IliasDirectoryFilter} -- A filter for directories. Will be applied on the
-                crawler level, these directories and all of their content is skipped.
-                (default: {lambdax:True})
-            transform {Transform} -- A transformation function for the output paths. Return None
-                to ignore a file. (default: {lambdax:x})
-            cookies {Optional[Path]} -- The path to store and load cookies from.
-                (default: {None})
-            username {Optional[str]} -- The SCC username. If none is given, it will prompt
-                the user. (default: {None})
-            password {Optional[str]} -- The SCC password. If none is given, it will prompt
-                the user. (default: {None})
-            download_strategy {DownloadStrategy} -- A function to determine which files need to
-                be downloaded. Can save bandwidth and reduce the number of requests.
-                (default: {download_modified_or_new})
-            clean {bool} -- Whether to clean up when the method finishes.
-            timeout {int} -- The download timeout for opencast videos. Sadly needed due to a
-                requests bug.
-        """
-        # This authenticator only works with the KIT ilias instance.
-        authenticator = KitShibbolethAuthenticator(username=username, password=password)
-        PRETTY.starting_synchronizer(target, "ILIAS", "An ILIAS element by url")
-
-        if not full_url.startswith("https://ilias.studium.kit.edu"):
-            raise FatalException("Not a valid KIT ILIAS URL")
-
-        organizer = self._ilias(
-            target=target,
-            base_url="https://ilias.studium.kit.edu/",
-            crawl_function=lambda crawler: crawler.recursive_crawl_url(full_url),
-            authenticator=authenticator,
-            cookies=cookies,
-            dir_filter=dir_filter,
-            transform=transform,
-            download_strategy=download_strategy,
-            clean=clean,
-            timeout=timeout
-        )
-
-        self._download_summary.merge(organizer.download_summary)
-
-        return organizer
-
-    @swallow_and_print_errors
-    def diva_kit(
-            self,
-            target: Union[PathLike, Organizer],
-            playlist_location: str,
-            transform: Transform = lambda x: x,
-            download_strategy: DivaDownloadStrategy = diva_download_new,
-            clean: bool = True
-    ) -> Organizer:
-        """
-        Synchronizes a folder with a DIVA playlist.
-
-        Arguments:
-            organizer {Organizer} -- The organizer to use.
-            playlist_location {str} -- the playlist id or the playlist URL
-              in the format 'https://mediaservice.bibliothek.kit.edu/#/details/DIVA-2019-271'
-
-        Keyword Arguments:
-            transform {Transform} -- A transformation function for the output paths. Return None
-                to ignore a file. (default: {lambdax:x})
-            download_strategy {DivaDownloadStrategy} -- A function to determine which files need to
-                be downloaded. Can save bandwidth and reduce the number of requests.
-                (default: {diva_download_new})
-            clean {bool} -- Whether to clean up when the method finishes.
-        """
-        tmp_dir = self._tmp_dir.new_subdir()
-
-        if playlist_location.startswith("http"):
-            playlist_id = DivaPlaylistCrawler.fetch_id(playlist_link=playlist_location)
+        crawlers: List[str]
+        if cli_crawlers is None:
+            log.explain("No crawlers specified on CLI")
+            log.explain("Running crawlers specified in config")
+            crawlers = self._find_config_crawlers(config)
         else:
-            playlist_id = playlist_location
+            log.explain("Crawlers specified on CLI")
+            crawlers = self._find_cli_crawlers(config, cli_crawlers)
 
-        if target is None:
-            PRETTY.starting_synchronizer("None", "DIVA", playlist_id)
-            raise FatalException("Got 'None' as target directory, aborting")
+        skips = {f"crawl:{name}" for name in cli_skips} if cli_skips else set()
+        for crawler in crawlers:
+            if crawler in skips:
+                log.explain(f"Skipping crawler {crawler!r}")
+        crawlers = [crawler for crawler in crawlers if crawler not in skips]
 
-        if isinstance(target, Organizer):
-            organizer = target
-        else:
-            organizer = Organizer(self.resolve(to_path(target)))
+        return crawlers
 
-        PRETTY.starting_synchronizer(organizer.path, "DIVA", playlist_id)
+    def _load_authenticators(self) -> None:
+        for name, section in self._config.auth_sections():
+            log.print(f"[bold bright_cyan]Loading[/] {escape(name)}")
 
-        crawler = DivaPlaylistCrawler(playlist_id)
-        downloader = DivaDownloader(tmp_dir, organizer, download_strategy)
+            auth_type = AuthSection(section).type()
+            authenticator_constructor = AUTHENTICATORS.get(auth_type)
+            if authenticator_constructor is None:
+                raise ConfigOptionError(name, "type", f"Unknown authenticator type: {auth_type!r}")
 
-        info = crawler.crawl()
+            authenticator = authenticator_constructor(name, section, self._config)
+            self._authenticators[name] = authenticator
 
-        transformed = apply_transform(transform, info)
-        if self._test_run:
-            self._print_transformables(transformed)
-            return organizer
+    def _load_crawlers(self) -> None:
+        # Cookie sharing
+        kit_ilias_web_paths: Dict[Authenticator, List[Path]] = {}
 
-        downloader.download_all(transformed)
+        for name, section in self._config.crawl_sections():
+            log.print(f"[bold bright_cyan]Loading[/] {escape(name)}")
 
-        if clean:
-            organizer.cleanup()
+            crawl_type = CrawlerSection(section).type()
+            crawler_constructor = CRAWLERS.get(crawl_type)
+            if crawler_constructor is None:
+                raise ConfigOptionError(name, "type", f"Unknown crawler type: {crawl_type!r}")
 
-        return organizer
+            crawler = crawler_constructor(name, section, self._config, self._authenticators)
+            self._crawlers[name] = crawler
+
+            if self._config.default_section.share_cookies():
+                if isinstance(crawler, KitIliasWebCrawler):
+                    crawler.share_cookies(kit_ilias_web_paths)
+
+    def debug_transforms(self) -> None:
+        for name in self._crawlers_to_run:
+            crawler = self._crawlers[name]
+            log.print("")
+            log.print(f"[bold bright_cyan]Debugging transforms[/] for {escape(name)}")
+            crawler.debug_transforms()
+
+    async def run(self, debug_transforms: bool) -> None:
+        """
+        May throw ConfigOptionError.
+        """
+
+        # These two functions must run inside the same event loop as the
+        # crawlers, so that any new objects (like Conditions or Futures) can
+        # obtain the correct event loop.
+        self._load_authenticators()
+        self._load_crawlers()
+
+        if debug_transforms:
+            log.output_explain = True
+            log.output_report = False
+            self.debug_transforms()
+            return
+
+        log.print("")
+
+        for name in self._crawlers_to_run:
+            crawler = self._crawlers[name]
+
+            log.print(f"[bold bright_cyan]Running[/] {escape(name)}")
+
+            try:
+                await crawler.run()
+            except (CrawlError, AuthError) as e:
+                log.error(str(e))
+            except Exception:
+                log.unexpected_exception()
+
+    def print_report(self) -> None:
+        for name in self._crawlers_to_run:
+            crawler = self._crawlers.get(name)
+            if crawler is None:
+                continue  # Crawler failed to load
+
+            log.report("")
+            log.report(f"[bold bright_cyan]Report[/] for {escape(name)}")
+
+            something_changed = False
+            for path in sorted(crawler.report.added_files):
+                something_changed = True
+                log.report(f"  [bold bright_green]Added[/] {fmt_path(path)}")
+            for path in sorted(crawler.report.changed_files):
+                something_changed = True
+                log.report(f"  [bold bright_yellow]Changed[/] {fmt_path(path)}")
+            for path in sorted(crawler.report.deleted_files):
+                something_changed = True
+                log.report(f"  [bold bright_magenta]Deleted[/] {fmt_path(path)}")
+            for path in sorted(crawler.report.not_deleted_files):
+                something_changed = True
+                log.report_not_deleted(f"  [bold bright_magenta]Not deleted[/] {fmt_path(path)}")
+
+            for warning in crawler.report.encountered_warnings:
+                something_changed = True
+                log.report(f"  [bold bright_red]Warning[/] {warning}")
+
+            for error in crawler.report.encountered_errors:
+                something_changed = True
+                log.report(f"  [bold bright_red]Error[/] {error}")
+
+            if not something_changed:
+                log.report("  Nothing changed")

PFERD/progress.py

@@ -1,111 +0,0 @@
-"""
-A small progress bar implementation.
-"""
-import sys
-from dataclasses import dataclass
-from types import TracebackType
-from typing import Optional, Type
-
-import requests
-from rich.console import Console
-from rich.progress import (BarColumn, DownloadColumn, Progress, TaskID,
-                           TextColumn, TimeRemainingColumn,
-                           TransferSpeedColumn)
-
-_progress: Progress = Progress(
-    TextColumn("[bold blue]{task.fields[name]}", justify="right"),
-    BarColumn(bar_width=None),
-    "[progress.percentage]{task.percentage:>3.1f}%",
-    "•",
-    DownloadColumn(),
-    "•",
-    TransferSpeedColumn(),
-    "•",
-    TimeRemainingColumn(),
-    console=Console(file=sys.stdout),
-    transient=True
-)
-
-
-def size_from_headers(response: requests.Response) -> Optional[int]:
-    """
-    Return the size of the download based on the response headers.
-
-    Arguments:
-        response {requests.Response} -- the response
-
-    Returns:
-        Optional[int] -- the size
-    """
-    if "Content-Length" in response.headers:
-        return int(response.headers["Content-Length"])
-    return None
-
-
-@dataclass
-class ProgressSettings:
-    """
-    Settings you can pass to customize the progress bar.
-    """
-    name: str
-    max_size: int
-
-
-def progress_for(settings: Optional[ProgressSettings]) -> 'ProgressContextManager':
-    """
-    Returns a context manager that displays progress
-
-    Returns:
-        ProgressContextManager -- the progress manager
-    """
-    return ProgressContextManager(settings)
-
-
-class ProgressContextManager:
-    """
-    A context manager used for displaying progress.
-    """
-
-    def __init__(self, settings: Optional[ProgressSettings]):
-        self._settings = settings
-        self._task_id: Optional[TaskID] = None
-
-    def __enter__(self) -> 'ProgressContextManager':
-        """Context manager entry function."""
-        if not self._settings:
-            return self
-
-        _progress.start()
-        self._task_id = _progress.add_task(
-            self._settings.name,
-            total=self._settings.max_size,
-            name=self._settings.name
-        )
-        return self
-
-    # pylint: disable=useless-return
-    def __exit__(
-            self,
-            exc_type: Optional[Type[BaseException]],
-            exc_value: Optional[BaseException],
-            traceback: Optional[TracebackType],
-    ) -> Optional[bool]:
-        """Context manager exit function. Removes the task."""
-        if self._task_id is None:
-            return None
-
-        _progress.remove_task(self._task_id)
-
-        if len(_progress.task_ids) == 0:
-            # We need to clean up after ourselves, as we were the last one
-            _progress.stop()
-            _progress.refresh()
-
-        return None
-
-    def advance(self, amount: float) -> None:
-        """
-        Advances the progress bar.
-        """
-        if self._task_id is not None:
-            _progress.advance(self._task_id, amount)

PFERD/report.py

@@ -0,0 +1,238 @@
+import json
+from pathlib import Path, PurePath
+from typing import Any, Dict, List, Optional, Set
+
+
+class ReportLoadError(Exception):
+    pass
+
+
+class MarkDuplicateError(Exception):
+    """
+    Tried to mark a file that was already marked.
+    """
+
+    def __init__(self, path: PurePath):
+        super().__init__(f"A previous file already used path {path}")
+        self.path = path
+
+
+class MarkConflictError(Exception):
+    """
+    Marking the path would have caused a conflict.
+
+    A conflict can have two reasons: Either the new file has the same path as
+    the parent directory of a known file, or a parent directory of the new file
+    has the same path as a known file. In either case, adding the new file
+    would require a file and a directory to share the same path, which is
+    usually not possible.
+    """
+
+    def __init__(self, path: PurePath, collides_with: PurePath):
+        super().__init__(f"File at {path} collides with previous file at {collides_with}")
+        self.path = path
+        self.collides_with = collides_with
+
+
+# TODO Use PurePath.is_relative_to when updating to 3.9
+def is_relative_to(a: PurePath, b: PurePath) -> bool:
+    try:
+        a.relative_to(b)
+        return True
+    except ValueError:
+        return False
+
+
+class Report:
+    """
+    A report of a synchronization. Includes all files found by the crawler, as
+    well as the set of changes made to local files.
+    """
+
+    def __init__(self) -> None:
+        # Paths found by the crawler, untransformed
+        self.found_paths: Set[PurePath] = set()
+
+        # Files reserved for metadata files (e. g. the report file or cookies)
+        # that can't be overwritten by user transforms and won't be cleaned up
+        # at the end.
+        self.reserved_files: Set[PurePath] = set()
+
+        # Files found by the crawler, transformed. Only includes files that
+        # were downloaded (or a download was attempted)
+        self.known_files: Set[PurePath] = set()
+
+        self.added_files: Set[PurePath] = set()
+        self.changed_files: Set[PurePath] = set()
+        self.deleted_files: Set[PurePath] = set()
+        # Files that should have been deleted by the cleanup but weren't
+        self.not_deleted_files: Set[PurePath] = set()
+
+        # Custom crawler-specific data
+        self.custom: Dict[str, Any] = dict()
+
+        # Encountered errors and warnings
+        self.encountered_warnings: List[str] = []
+        self.encountered_errors: List[str] = []
+
+    @staticmethod
+    def _get_list_of_strs(data: Dict[str, Any], key: str) -> List[str]:
+        result: Any = data.get(key, [])
+
+        if not isinstance(result, list):
+            raise ReportLoadError(f"Incorrect format: {key!r} is not a list")
+
+        for elem in result:
+            if not isinstance(elem, str):
+                raise ReportLoadError(f"Incorrect format: {key!r} must contain only strings")
+
+        return result
+
+    @staticmethod
+    def _get_str_dictionary(data: Dict[str, Any], key: str) -> Dict[str, Any]:
+        result: Dict[str, Any] = data.get(key, {})
+
+        if not isinstance(result, dict):
+            raise ReportLoadError(f"Incorrect format: {key!r} is not a dictionary")
+
+        return result
+
+    @classmethod
+    def load(cls, path: Path) -> "Report":
+        """
+        May raise OSError, UnicodeDecodeError, JsonDecodeError, ReportLoadError.
+        """
+
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+
+        if not isinstance(data, dict):
+            raise ReportLoadError("Incorrect format: Root is not an object")
+
+        self = cls()
+        for elem in self._get_list_of_strs(data, "found"):
+            self.found(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "reserved"):
+            self.mark_reserved(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "known"):
+            self.mark(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "added"):
+            self.add_file(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "changed"):
+            self.change_file(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "deleted"):
+            self.delete_file(PurePath(elem))
+        for elem in self._get_list_of_strs(data, "not_deleted"):
+            self.not_delete_file(PurePath(elem))
+        self.custom = self._get_str_dictionary(data, "custom")
+        self.encountered_errors = self._get_list_of_strs(data, "encountered_errors")
+        self.encountered_warnings = self._get_list_of_strs(data, "encountered_warnings")
+
+        return self
+
+    def store(self, path: Path) -> None:
+        """
+        May raise OSError.
+        """
+
+        data = {
+            "found": [str(path) for path in sorted(self.found_paths)],
+            "reserved": [str(path) for path in sorted(self.reserved_files)],
+            "known": [str(path) for path in sorted(self.known_files)],
+            "added": [str(path) for path in sorted(self.added_files)],
+            "changed": [str(path) for path in sorted(self.changed_files)],
+            "deleted": [str(path) for path in sorted(self.deleted_files)],
+            "not_deleted": [str(path) for path in sorted(self.not_deleted_files)],
+            "custom": self.custom,
+            "encountered_warnings": self.encountered_warnings,
+            "encountered_errors": self.encountered_errors,
+        }
+
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, sort_keys=True)
+            f.write("\n")  # json.dump doesn't do this
+
+    def found(self, path: PurePath) -> None:
+        self.found_paths.add(path)
+
+    def mark_reserved(self, path: PurePath) -> None:
+        if path in self.marked:
+            raise RuntimeError("Trying to reserve an already reserved file")
+
+        self.reserved_files.add(path)
+
+    def mark(self, path: PurePath) -> None:
+        """
+        Mark a previously unknown file as known.
+
+        May throw a MarkDuplicateError or a MarkConflictError. For more detail,
+        see the respective exception's docstring.
+        """
+
+        for other in self.marked:
+            if path == other:
+                raise MarkDuplicateError(path)
+
+            if is_relative_to(path, other) or is_relative_to(other, path):
+                raise MarkConflictError(path, other)
+
+        self.known_files.add(path)
+
+    @property
+    def marked(self) -> Set[PurePath]:
+        return self.known_files | self.reserved_files
+
+    def is_marked(self, path: PurePath) -> bool:
+        return path in self.marked
+
+    def add_file(self, path: PurePath) -> None:
+        """
+        Unlike mark(), this function accepts any paths.
+        """
+
+        self.added_files.add(path)
+
+    def change_file(self, path: PurePath) -> None:
+        """
+        Unlike mark(), this function accepts any paths.
+        """
+
+        self.changed_files.add(path)
+
+    def delete_file(self, path: PurePath) -> None:
+        """
+        Unlike mark(), this function accepts any paths.
+        """
+
+        self.deleted_files.add(path)
+
+    def not_delete_file(self, path: PurePath) -> None:
+        """
+        Unlike mark(), this function accepts any paths.
+        """
+
+        self.not_deleted_files.add(path)
+
+    def add_custom_value(self, key: str, value: Any) -> None:
+        """
+        Adds a custom value under the passed key, overwriting any existing
+        """
+        self.custom[key] = value
+
+    def get_custom_value(self, key: str) -> Optional[Any]:
+        """
+        Retrieves a custom value for the given key.
+        """
+        return self.custom.get(key)
+
+    def add_error(self, error: str) -> None:
+        """
+        Adds an error to this report's error list.
+        """
+        self.encountered_errors.append(error)
+
+    def add_warning(self, warning: str) -> None:
+        """
+        Adds a warning to this report's warning list.
+        """
+        self.encountered_warnings.append(warning)

PFERD/tmp_dir.py

@@ -1,79 +0,0 @@
-"""Helper functions and classes for temporary folders."""
-
-import logging
-import shutil
-from pathlib import Path
-from types import TracebackType
-from typing import Optional, Type
-
-from .location import Location
-
-LOGGER = logging.getLogger(__name__)
-
-
-class TmpDir(Location):
-    """A temporary folder that can create files or nested temp folders."""
-
-    def __init__(self, path: Path):
-        """Create a new temporary folder for the given path."""
-        super().__init__(path)
-        self._counter = 0
-        self.cleanup()
-        self.path.mkdir(parents=True, exist_ok=True)
-
-    def __str__(self) -> str:
-        """Format the folder as a string."""
-        return f"Folder at {self.path}"
-
-    def __enter__(self) -> 'TmpDir':
-        """Context manager entry function."""
-        return self
-
-    # pylint: disable=useless-return
-    def __exit__(
-            self,
-            exc_type: Optional[Type[BaseException]],
-            exc_value: Optional[BaseException],
-            traceback: Optional[TracebackType],
-    ) -> Optional[bool]:
-        """Context manager exit function. Calls cleanup()."""
-        self.cleanup()
-        return None
-
-    def new_path(self, prefix: Optional[str] = None) -> Path:
-        """
-        Return a unique path inside the directory. Doesn't create a file or
-        directory.
-        """
-
-        name = f"{prefix if prefix else 'tmp'}-{self._inc_and_get_counter():03}"
-
-        LOGGER.debug("Creating temp file %s", name)
-
-        return self.resolve(Path(name))
-
-    def new_subdir(self, prefix: Optional[str] = None) -> 'TmpDir':
-        """
-        Create a new nested temporary folder and return it.
-        """
-
-        name = f"{prefix if prefix else 'tmp'}-{self._inc_and_get_counter():03}"
-        sub_path = self.resolve(Path(name))
-        sub_path.mkdir(parents=True)
-
-        LOGGER.debug("Creating temp dir %s at %s", name, sub_path)
-
-        return TmpDir(sub_path)
-
-    def cleanup(self) -> None:
-        """Delete this folder and all contained files."""
-        LOGGER.debug("Deleting temp folder %s", self.path)
-
-        if self.path.resolve().exists():
-            shutil.rmtree(self.path.resolve())
-
-    def _inc_and_get_counter(self) -> int:
-        """Get and increment the counter by one."""
-        counter = self._counter
-        self._counter += 1
-        return counter

PFERD/transform.py

@@ -1,127 +0,0 @@
-"""
-Transforms let the user define functions to decide where the downloaded files
-should be placed locally. They let the user do more advanced things like moving
-only files whose names match a regex, or renaming files from one numbering
-scheme to another.
-"""
-
-from dataclasses import dataclass
-from pathlib import PurePath
-from typing import Callable, List, Optional, TypeVar
-
-from .utils import PathLike, Regex, to_path, to_pattern
-
-Transform = Callable[[PurePath], Optional[PurePath]]
-
-
-@dataclass
-class Transformable:
-    """
-    An object that can be transformed by a Transform.
-    """
-
-    path: PurePath
-
-
-TF = TypeVar("TF", bound=Transformable)
-
-
-def apply_transform(
-        transform: Transform,
-        transformables: List[TF],
-) -> List[TF]:
-    """
-    Apply a Transform to multiple Transformables, discarding those that were
-    not transformed by the Transform.
-    """
-
-    result: List[TF] = []
-    for transformable in transformables:
-        new_path = transform(transformable.path)
-        if new_path:
-            transformable.path = new_path
-            result.append(transformable)
-    return result
-
-# Transform combinators
-
-keep = lambda path: path
-
-def attempt(*args: Transform) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        for transform in args:
-            result = transform(path)
-            if result:
-                return result
-        return None
-    return inner
-
-def optionally(transform: Transform) -> Transform:
-    return attempt(transform, lambda path: path)
-
-def do(*args: Transform) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        current = path
-        for transform in args:
-            result = transform(current)
-            if result:
-                current = result
-            else:
-                return None
-        return current
-    return inner
-
-def predicate(pred: Callable[[PurePath], bool]) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        if pred(path):
-            return path
-        return None
-    return inner
-
-def glob(pattern: str) -> Transform:
-    return predicate(lambda path: path.match(pattern))
-
-def move_dir(source_dir: PathLike, target_dir: PathLike) -> Transform:
-    source_path = to_path(source_dir)
-    target_path = to_path(target_dir)
-    def inner(path: PurePath) -> Optional[PurePath]:
-        if source_path in path.parents:
-            return target_path / path.relative_to(source_path)
-        return None
-    return inner
-
-def move(source: PathLike, target: PathLike) -> Transform:
-    source_path = to_path(source)
-    target_path = to_path(target)
-    def inner(path: PurePath) -> Optional[PurePath]:
-        if path == source_path:
-            return target_path
-        return None
-    return inner
-
-def rename(source: str, target: str) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        if path.name == source:
-            return path.with_name(target)
-        return None
-    return inner
-
-def re_move(regex: Regex, target: str) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        match = to_pattern(regex).fullmatch(str(path))
-        if match:
-            groups = [match.group(0)]
-            groups.extend(match.groups())
-            return PurePath(target.format(*groups))
-        return None
-    return inner
-
-def re_rename(regex: Regex, target: str) -> Transform:
-    def inner(path: PurePath) -> Optional[PurePath]:
-        match = to_pattern(regex).fullmatch(path.name)
-        if match:
-            groups = [match.group(0)]
-            groups.extend(match.groups())
-            return path.with_name(target.format(*groups))
-        return None
-    return inner

PFERD/transformer.py

@@ -0,0 +1,439 @@
+import ast
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import PurePath
+from typing import Callable, Dict, List, Optional, Sequence, TypeVar, Union
+
+from .logging import log
+from .utils import fmt_path, str_path
+
+
+class ArrowHead(Enum):
+    NORMAL = 0
+    SEQUENCE = 1
+
+
+class Ignore:
+    pass
+
+
+class Empty:
+    pass
+
+
+RightSide = Union[str, Ignore, Empty]
+
+
+@dataclass
+class Transformed:
+    path: PurePath
+
+
+class Ignored:
+    pass
+
+
+TransformResult = Optional[Union[Transformed, Ignored]]
+
+
+@dataclass
+class Rule:
+    left: str
+    left_index: int
+    name: str
+    head: ArrowHead
+    right: RightSide
+    right_index: int
+
+    def right_result(self, path: PurePath) -> Union[str, Transformed, Ignored]:
+        if isinstance(self.right, str):
+            return self.right
+        elif isinstance(self.right, Ignore):
+            return Ignored()
+        elif isinstance(self.right, Empty):
+            return Transformed(path)
+        else:
+            raise RuntimeError(f"Right side has invalid type {type(self.right)}")
+
+
+class Transformation(ABC):
+    def __init__(self, rule: Rule):
+        self.rule = rule
+
+    @abstractmethod
+    def transform(self, path: PurePath) -> TransformResult:
+        pass
+
+
+class ExactTf(Transformation):
+    def transform(self, path: PurePath) -> TransformResult:
+        if path != PurePath(self.rule.left):
+            return None
+
+        right = self.rule.right_result(path)
+        if not isinstance(right, str):
+            return right
+
+        return Transformed(PurePath(right))
+
+
+class ExactReTf(Transformation):
+    def transform(self, path: PurePath) -> TransformResult:
+        match = re.fullmatch(self.rule.left, str_path(path))
+        if not match:
+            return None
+
+        right = self.rule.right_result(path)
+        if not isinstance(right, str):
+            return right
+
+        # For some reason, mypy thinks that "groups" has type List[str]. But
+        # since elements of "match.groups()" can be None, mypy is wrong.
+        groups: Sequence[Optional[str]] = [match[0]] + list(match.groups())
+
+        locals_dir: Dict[str, Union[str, int, float]] = {}
+        for i, group in enumerate(groups):
+            if group is None:
+                continue
+
+            locals_dir[f"g{i}"] = group
+
+            try:
+                locals_dir[f"i{i}"] = int(group)
+            except ValueError:
+                pass
+
+            try:
+                locals_dir[f"f{i}"] = float(group)
+            except ValueError:
+                pass
+
+        result = eval(f"f{right!r}", {}, locals_dir)
+        return Transformed(PurePath(result))
+
+
+class RenamingParentsTf(Transformation):
+    def __init__(self, sub_tf: Transformation):
+        super().__init__(sub_tf.rule)
+        self.sub_tf = sub_tf
+
+    def transform(self, path: PurePath) -> TransformResult:
+        for i in range(len(path.parts), -1, -1):
+            parent = PurePath(*path.parts[:i])
+            child = PurePath(*path.parts[i:])
+
+            transformed = self.sub_tf.transform(parent)
+            if not transformed:
+                continue
+            elif isinstance(transformed, Transformed):
+                return Transformed(transformed.path / child)
+            elif isinstance(transformed, Ignored):
+                return transformed
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(transformed)}: {transformed}")
+
+        return None
+
+
+class RenamingPartsTf(Transformation):
+    def __init__(self, sub_tf: Transformation):
+        super().__init__(sub_tf.rule)
+        self.sub_tf = sub_tf
+
+    def transform(self, path: PurePath) -> TransformResult:
+        result = PurePath()
+        any_part_matched = False
+        for part in path.parts:
+            transformed = self.sub_tf.transform(PurePath(part))
+            if not transformed:
+                result /= part
+            elif isinstance(transformed, Transformed):
+                result /= transformed.path
+                any_part_matched = True
+            elif isinstance(transformed, Ignored):
+                return transformed
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(transformed)}: {transformed}")
+
+        if any_part_matched:
+            return Transformed(result)
+        else:
+            return None
+
+
+class RuleParseError(Exception):
+    def __init__(self, line: "Line", reason: str):
+        super().__init__(f"Error in rule on line {line.line_nr}, column {line.index}: {reason}")
+
+        self.line = line
+        self.reason = reason
+
+    def pretty_print(self) -> None:
+        log.error(f"Error parsing rule on line {self.line.line_nr}:")
+        log.error_contd(self.line.line)
+        spaces = " " * self.line.index
+        log.error_contd(f"{spaces}^--- {self.reason}")
+
+
+T = TypeVar("T")
+
+
+class Line:
+    def __init__(self, line: str, line_nr: int):
+        self._line = line
+        self._line_nr = line_nr
+        self._index = 0
+
+    @property
+    def line(self) -> str:
+        return self._line
+
+    @property
+    def line_nr(self) -> int:
+        return self._line_nr
+
+    @property
+    def index(self) -> int:
+        return self._index
+
+    @index.setter
+    def index(self, index: int) -> None:
+        self._index = index
+
+    @property
+    def rest(self) -> str:
+        return self.line[self.index:]
+
+    def peek(self, amount: int = 1) -> str:
+        return self.rest[:amount]
+
+    def take(self, amount: int = 1) -> str:
+        string = self.peek(amount)
+        self.index += len(string)
+        return string
+
+    def expect(self, string: str) -> str:
+        if self.peek(len(string)) == string:
+            return self.take(len(string))
+        else:
+            raise RuleParseError(self, f"Expected {string!r}")
+
+    def expect_with(self, string: str, value: T) -> T:
+        self.expect(string)
+        return value
+
+    def one_of(self, parsers: List[Callable[[], T]], description: str) -> T:
+        for parser in parsers:
+            index = self.index
+            try:
+                return parser()
+            except RuleParseError:
+                self.index = index
+
+        raise RuleParseError(self, description)
+
+
+# RULE = LEFT SPACE '-' NAME '-' HEAD (SPACE RIGHT)?
+# SPACE = ' '+
+# NAME = '' | 'exact' | 'name' | 're' | 'exact-re' | 'name-re'
+# HEAD = '>' | '>>'
+# LEFT = STR | QUOTED_STR
+# RIGHT = STR | QUOTED_STR | '!'
+
+
+def parse_zero_or_more_spaces(line: Line) -> None:
+    while line.peek() == " ":
+        line.take()
+
+
+def parse_one_or_more_spaces(line: Line) -> None:
+    line.expect(" ")
+    parse_zero_or_more_spaces(line)
+
+
+def parse_str(line: Line) -> str:
+    result = []
+    while c := line.peek():
+        if c == " ":
+            break
+        else:
+            line.take()
+            result.append(c)
+
+    if result:
+        return "".join(result)
+    else:
+        raise RuleParseError(line, "Expected non-space character")
+
+
+QUOTATION_MARKS = {'"', "'"}
+
+
+def parse_quoted_str(line: Line) -> str:
+    escaped = False
+
+    # Points to first character of string literal
+    start_index = line.index
+
+    quotation_mark = line.peek()
+    if quotation_mark not in QUOTATION_MARKS:
+        raise RuleParseError(line, "Expected quotation mark")
+    line.take()
+
+    while c := line.peek():
+        if escaped:
+            escaped = False
+            line.take()
+        elif c == quotation_mark:
+            line.take()
+            stop_index = line.index
+            literal = line.line[start_index:stop_index]
+            try:
+                return ast.literal_eval(literal)
+            except SyntaxError as e:
+                line.index = start_index
+                raise RuleParseError(line, str(e)) from e
+        elif c == "\\":
+            escaped = True
+            line.take()
+        else:
+            line.take()
+
+    raise RuleParseError(line, "Expected end of string literal")
+
+
+def parse_left(line: Line) -> str:
+    if line.peek() in QUOTATION_MARKS:
+        return parse_quoted_str(line)
+    else:
+        return parse_str(line)
+
+
+def parse_right(line: Line) -> Union[str, Ignore]:
+    c = line.peek()
+    if c in QUOTATION_MARKS:
+        return parse_quoted_str(line)
+    else:
+        string = parse_str(line)
+        if string == "!":
+            return Ignore()
+        return string
+
+
+def parse_arrow_name(line: Line) -> str:
+    return line.one_of([
+        lambda: line.expect("exact-re"),
+        lambda: line.expect("exact"),
+        lambda: line.expect("name-re"),
+        lambda: line.expect("name"),
+        lambda: line.expect("re"),
+        lambda: line.expect(""),
+    ], "Expected arrow name")
+
+
+def parse_arrow_head(line: Line) -> ArrowHead:
+    return line.one_of([
+        lambda: line.expect_with(">>", ArrowHead.SEQUENCE),
+        lambda: line.expect_with(">", ArrowHead.NORMAL),
+    ], "Expected arrow head")
+
+
+def parse_eol(line: Line) -> None:
+    if line.peek():
+        raise RuleParseError(line, "Expected end of line")
+
+
+def parse_rule(line: Line) -> Rule:
+    parse_zero_or_more_spaces(line)
+    left_index = line.index
+    left = parse_left(line)
+
+    parse_one_or_more_spaces(line)
+
+    line.expect("-")
+    name = parse_arrow_name(line)
+    line.expect("-")
+    head = parse_arrow_head(line)
+
+    right_index = line.index
+    right: RightSide
+    try:
+        parse_zero_or_more_spaces(line)
+        parse_eol(line)
+        right = Empty()
+    except RuleParseError:
+        line.index = right_index
+        parse_one_or_more_spaces(line)
+        right = parse_right(line)
+        parse_eol(line)
+
+    return Rule(left, left_index, name, head, right, right_index)
+
+
+def parse_transformation(line: Line) -> Transformation:
+    rule = parse_rule(line)
+
+    if rule.name == "":
+        return RenamingParentsTf(ExactTf(rule))
+    elif rule.name == "exact":
+        return ExactTf(rule)
+    elif rule.name == "name":
+        if len(PurePath(rule.left).parts) > 1:
+            line.index = rule.left_index
+            raise RuleParseError(line, "Expected name, not multiple segments")
+        return RenamingPartsTf(ExactTf(rule))
+    elif rule.name == "re":
+        return RenamingParentsTf(ExactReTf(rule))
+    elif rule.name == "exact-re":
+        return ExactReTf(rule)
+    elif rule.name == "name-re":
+        return RenamingPartsTf(ExactReTf(rule))
+    else:
+        raise RuntimeError(f"Invalid arrow name {rule.name!r}")
+
+
+class Transformer:
+    def __init__(self, rules: str):
+        """
+        May throw a RuleParseException.
+        """
+
+        self._tfs = []
+        for i, line in enumerate(rules.split("\n")):
+            line = line.strip()
+            if line:
+                tf = parse_transformation(Line(line, i))
+                self._tfs.append((line, tf))
+
+    def transform(self, path: PurePath) -> Optional[PurePath]:
+        for i, (line, tf) in enumerate(self._tfs):
+            log.explain(f"Testing rule {i+1}: {line}")
+
+            try:
+                result = tf.transform(path)
+            except Exception as e:
+                log.warn(f"Error while testing rule {i+1}: {line}")
+                log.warn_contd(str(e))
+                continue
+
+            if not result:
+                continue
+
+            if isinstance(result, Ignored):
+                log.explain("Match found, path ignored")
+                return None
+
+            if tf.rule.head == ArrowHead.NORMAL:
+                log.explain(f"Match found, transformed path to {fmt_path(result.path)}")
+                path = result.path
+                break
+            elif tf.rule.head == ArrowHead.SEQUENCE:
+                log.explain(f"Match found, updated path to {fmt_path(result.path)}")
+                path = result.path
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(result)}: {result}")
+
+        log.explain(f"Final result: {fmt_path(path)}")
+        return path

PFERD/utils.py

@@ -1,98 +1,144 @@
-"""
-A few utility bobs and bits.
-"""
-
-import re
+import asyncio
+import getpass
+import sys
+import threading
+from abc import ABC, abstractmethod
+from contextlib import AsyncExitStack
 from pathlib import Path, PurePath
-from typing import Optional, Tuple, Union
+from types import TracebackType
+from typing import Any, Callable, Dict, Generic, Optional, Type, TypeVar
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
 
 import bs4
-import requests
 
-from .progress import ProgressSettings, progress_for, size_from_headers
-
-PathLike = Union[PurePath, str, Tuple[str, ...]]
+T = TypeVar("T")
 
 
-def to_path(pathlike: PathLike) -> Path:
+async def in_daemon_thread(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
+    loop = asyncio.get_running_loop()
+    future: asyncio.Future[T] = asyncio.Future()
+
+    def thread_func() -> None:
+        result = func()
+        loop.call_soon_threadsafe(future.set_result, result)
+
+    threading.Thread(target=thread_func, daemon=True).start()
+
+    return await future
+
+
+async def ainput(prompt: str) -> str:
+    return await in_daemon_thread(lambda: input(prompt))
+
+
+async def agetpass(prompt: str) -> str:
+    return await in_daemon_thread(lambda: getpass.getpass(prompt))
+
+
+async def prompt_yes_no(query: str, default: Optional[bool]) -> bool:
     """
-    Convert a given PathLike into a Path.
-    """
-    if isinstance(pathlike, tuple):
-        return Path(*pathlike)
-    return Path(pathlike)
-
-
-Regex = Union[str, re.Pattern]
-
-
-def to_pattern(regex: Regex) -> re.Pattern:
-    """
-    Convert a regex to a re.Pattern.
-    """
-    if isinstance(regex, re.Pattern):
-        return regex
-    return re.compile(regex)
-
-
-def soupify(response: requests.Response) -> bs4.BeautifulSoup:
-    """
-    Wrap a requests response in a bs4 object.
-    """
-
-    return bs4.BeautifulSoup(response.text, "html.parser")
-
-
-def stream_to_path(
-        response: requests.Response,
-        target: Path,
-        progress_name: Optional[str] = None,
-        chunk_size: int = 1024 ** 2
-) -> None:
-    """
-    Download a requests response content to a file by streaming it. This
-    function avoids excessive memory usage when downloading large files. The
-    chunk_size is in bytes.
-
-    If progress_name is None, no progress bar will be shown. Otherwise a progress
-    bar will appear, if the download is bigger than an internal threshold.
-    """
-
-    with response:
-        length = size_from_headers(response)
-        if progress_name and length and int(length) > 1024 * 1024 * 10:  # 10 MiB
-            settings: Optional[ProgressSettings] = ProgressSettings(progress_name, length)
-        else:
-            settings = None
-
-        with open(target, 'wb') as file_descriptor:
-            with progress_for(settings) as progress:
-                for chunk in response.iter_content(chunk_size=chunk_size):
-                    file_descriptor.write(chunk)
-                    progress.advance(len(chunk))
-
-
-def prompt_yes_no(question: str, default: Optional[bool] = None) -> bool:
-    """
-    Prompts the user a yes/no question and returns their choice.
+    Asks the user a yes/no question and returns their choice.
     """
 
     if default is True:
-        prompt = "[Y/n]"
+        query += " [Y/n] "
     elif default is False:
-        prompt = "[y/N]"
+        query += " [y/N] "
     else:
-        prompt = "[y/n]"
-
-    text = f"{question} {prompt} "
-    wrong_reply = "Please reply with 'yes'/'y' or 'no'/'n'."
+        query += " [y/n] "
 
     while True:
-        response = input(text).strip().lower()
-        if response in {"yes", "ye", "y"}:
+        response = (await ainput(query)).strip().lower()
+        if response == "y":
             return True
-        if response in {"no", "n"}:
+        elif response == "n":
             return False
-        if response == "" and default is not None:
+        elif response == "" and default is not None:
             return default
-        print(wrong_reply)
+
+        print("Please answer with 'y' or 'n'.")
+
+
+def soupify(data: bytes) -> bs4.BeautifulSoup:
+    """
+    Parses HTML to a beautifulsoup object.
+    """
+
+    return bs4.BeautifulSoup(data, "html.parser")
+
+
+def url_set_query_param(url: str, param: str, value: str) -> str:
+    """
+    Set a query parameter in an url, overwriting existing ones with the same name.
+    """
+    scheme, netloc, path, query, fragment = urlsplit(url)
+    query_parameters = parse_qs(query)
+    query_parameters[param] = [value]
+    new_query_string = urlencode(query_parameters, doseq=True)
+
+    return urlunsplit((scheme, netloc, path, new_query_string, fragment))
+
+
+def url_set_query_params(url: str, params: Dict[str, str]) -> str:
+    """
+    Sets multiple query parameters in an url, overwriting existing ones.
+    """
+    result = url
+
+    for key, val in params.items():
+        result = url_set_query_param(result, key, val)
+
+    return result
+
+
+def str_path(path: PurePath) -> str:
+    if not path.parts:
+        return "."
+    return "/".join(path.parts)
+
+
+def fmt_path(path: PurePath) -> str:
+    return repr(str_path(path))
+
+
+def fmt_real_path(path: Path) -> str:
+    return repr(str(path.absolute()))
+
+
+class ReusableAsyncContextManager(ABC, Generic[T]):
+    def __init__(self) -> None:
+        self._active = False
+        self._stack = AsyncExitStack()
+
+    @abstractmethod
+    async def _on_aenter(self) -> T:
+        pass
+
+    async def __aenter__(self) -> T:
+        if self._active:
+            raise RuntimeError("Nested or otherwise concurrent usage is not allowed")
+
+        self._active = True
+        await self._stack.__aenter__()
+
+        # See https://stackoverflow.com/a/13075071
+        try:
+            result: T = await self._on_aenter()
+        except:  # noqa: E722 do not use bare 'except'
+            if not await self.__aexit__(*sys.exc_info()):
+                raise
+
+        return result
+
+    async def __aexit__(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_value: Optional[BaseException],
+            traceback: Optional[TracebackType],
+    ) -> Optional[bool]:
+        if not self._active:
+            raise RuntimeError("__aexit__ called too many times")
+
+        result = await self._stack.__aexit__(exc_type, exc_value, traceback)
+        self._active = False
+        return result

PFERD/version.py

@@ -0,0 +1,2 @@
+NAME = "PFERD"
+VERSION = "3.5.0"

README.md

@@ -2,251 +2,146 @@
 
 **P**rogramm zum **F**lotten, **E**infachen **R**unterladen von **D**ateien
 
-- [Quickstart with `sync_url`](#quickstart-with-sync_url)
-- [Installation](#installation)
-    - [Upgrading from 2.0.0 to 2.1.0+](#upgrading-from-200-to-210)
-- [Example setup](#example-setup)
-- [Usage](#usage)
-    - [General concepts](#general-concepts)
-    - [Constructing transforms](#constructing-transforms)
-        - [Transform creators](#transform-creators)
-        - [Transform combinators](#transform-combinators)
-    - [A short, but commented example](#a-short-but-commented-example)
+Other resources:
 
-## Quickstart with `sync_url`
-
-The `sync_url` program allows you to just synchronize a given ILIAS URL (of a
-course, a folder, your personal desktop, etc.) without any extra configuration
-or setting up. Download the program, open ILIAS, copy the URL from the address
-bar and pass it to sync_url.
-
-It bundles everything it needs in one executable and is easy to
-use, but doesn't expose all the configuration options and tweaks a full install
-does.
-
-1. Download the `sync_url` binary from the [latest release](https://github.com/Garmelon/PFERD/releases/latest).
-2. Run the binary in your terminal (`./sync_url` or `sync_url.exe` in the CMD) to see the help and use it
+- [Config file format](CONFIG.md)
+- [Changelog](CHANGELOG.md)
+- [Development Guide](DEV.md)
 
 ## Installation
 
-Ensure that you have at least Python 3.8 installed.
+### Direct download
+
+Binaries for Linux, Windows and Mac can be downloaded directly from the
+[latest release](https://github.com/Garmelon/PFERD/releases/latest).
+
+### With pip
+
+Ensure you have at least Python 3.9 installed. Run the following command to
+install PFERD or upgrade it to the latest version:
 
-To install PFERD or update your installation to the latest version, run this
-wherever you want to install or have already installed PFERD:
 ```
-$ pip install git+https://github.com/Garmelon/PFERD@v2.4.0
+$ pip install --upgrade git+https://github.com/Garmelon/PFERD@latest
 ```
 
-The use of [venv] is recommended.
+The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
 
-[venv]: https://docs.python.org/3/library/venv.html
+### With package managers
 
-### Upgrading from 2.0.0 to 2.1.0+
+Unofficial packages are available for:
+- [AUR](https://aur.archlinux.org/packages/pferd)
+- [brew](https://formulae.brew.sh/formula/pferd)
+- [conda-forge](https://github.com/conda-forge/pferd-feedstock)
+- [nixpkgs](https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/misc/pferd/default.nix)
+- [PyPi](https://pypi.org/project/pferd)
 
-- The `IliasDirectoryType` type was renamed to `IliasElementType` and is now far more detailed.
-  The new values are: `REGULAR_FOLDER`, `VIDEO_FOLDER`, `EXERCISE_FOLDER`, `REGULAR_FILE`, `VIDEO_FILE`, `FORUM`, `EXTERNAL_LINK`.
-- Forums and external links are skipped automatically if you use the `kit_ilias` helper.
+See also PFERD's [repology page](https://repology.org/project/pferd/versions).
 
-## Example setup
+## Basic usage
 
-In this example, `python3` refers to at least Python 3.8.
+PFERD can be run directly from the command line with no config file. Run `pferd
+-h` to get an overview of available commands and options. Run `pferd <command>
+-h` to see which options a command has.
+
+For example, you can download your personal desktop from the KIT ILIAS like
+this:
 
-A full example setup and initial use could look like:
 ```
-$ mkdir Vorlesungen
-$ cd Vorlesungen
-$ python3 -m venv .venv
-$ .venv/bin/activate
-$ pip install git+https://github.com/Garmelon/PFERD@v2.4.0
-$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.4.0/example_config.py
-$ python3 example_config.py
-$ deactivate
+$ pferd kit-ilias-web desktop <output_directory>
 ```
 
-Subsequent runs of the program might look like:
+Also, you can download most ILIAS pages directly like this:
+
 ```
-$ cd Vorlesungen
-$ .venv/bin/activate
-$ python3 example_config.py
-$ deactivate
+$ pferd kit-ilias-web <url> <output_directory>
 ```
 
-If you just want to get started and crawl *your entire ILIAS Desktop* instead
-of a given set of courses, please replace `example_config.py` with
-`example_config_personal_desktop.py` in all of the instructions below (`curl` call and
-`python3` run command).
+However, the CLI only lets you download a single thing at a time, and the
+resulting command can grow long quite quickly. Because of this, PFERD can also
+be used with a config file.
 
-## Usage
+To get started, just take a command you've been using and add `--dump-config`
+directly after `pferd`, like this:
 
-### General concepts
-
-A PFERD config is a normal python file that starts multiple *synchronizers*
-which do all the heavy lifting. While you can create and wire them up manually,
-you are encouraged to use the helper methods provided in `PFERD.Pferd`.
-
-The synchronizers take some input arguments specific to their service and a
-*transform*. The transform receives the computed path of an element in ILIAS and
-can return either an output path (so you can rename files or move them around as
-you wish) or `None` if you do not want to save the given file.
-
-Additionally the ILIAS synchronizer allows you to define a *crawl filter*. This
-filter also receives the computed path as the input, but is only called for
-*directories*. If you return `True`, the directory will be crawled and
-searched. If you return `False` the directory will be ignored and nothing in it
-will be passed to the transform.
-
-### Constructing transforms
-
-While transforms are just normal python functions, writing them by hand can
-quickly become tedious. In order to help you with writing your own transforms
-and filters, PFERD defines a few useful transform creators and combinators in
-the `PFERD.transform` module:
-
-#### Transform creators
-
-These methods let you create a few basic transform building blocks:
-
-- **`glob(glob)`**  
-  Creates a transform that returns the unchanged path if the glob matches the path and `None` otherwise.
-  See also [Path.match].  
-  Example: `glob("Übung/*.pdf")`
-- **`predicate(pred)`**  
-  Creates a transform that returns the unchanged path if `pred(path)` returns a truthy value.
-  Returns `None` otherwise.  
-  Example: `predicate(lambda path: len(path.parts) == 3)`
-- **`move_dir(source, target)`**  
-  Creates a transform that moves all files from the `source` to the `target` directory.  
-  Example: `move_dir("Übung/", "Blätter/")`
-- **`move(source, target)`**  
-  Creates a transform that moves the `source` file to `target`.  
-  Example: `move("Vorlesung/VL02_Automten.pdf", "Vorlesung/VL02_Automaten.pdf")`
-- **`rename(source, target)`**  
-  Creates a transform that renames all files named `source` to `target`.
-  This transform works on the file names, not paths, and thus works no matter where the file is located.  
-  Example: `rename("VL02_Automten.pdf", "VL02_Automaten.pdf")`
-- **`re_move(regex, target)`**  
-  Creates a transform that moves all files matching `regex` to `target`.
-  The transform `str.format` on the `target` string with the contents of the capturing groups before returning it.
-  The capturing groups can be accessed via their index.
-  See also [Match.group].  
-  Example: `re_move(r"Übung/Blatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf")`
-- **`re_rename(regex, target)`**  
-  Creates a transform that renames all files matching `regex` to `target`.
-  This transform works on the file names, not paths, and thus works no matter where the file is located.  
-  Example: `re_rename(r"VL(\d+)(.*)\.pdf", "Vorlesung_Nr_{1}__{2}.pdf")`
-
-All movement or rename transforms above return `None` if a file doesn't match
-their movement or renaming criteria. This enables them to be used as building
-blocks to build up more complex transforms.
-
-In addition, `PFERD.transform` also defines the `keep` transform which returns its input path unchanged.
-This behaviour can be very useful when creating more complex transforms.
-See below for example usage.
-
-[Path.match]: https://docs.python.org/3/library/pathlib.html#pathlib.Path.match
-[Match.group]: https://docs.python.org/3/library/re.html#re.Match.group
-
-#### Transform combinators
-
-These methods let you combine transforms into more complex transforms:
-
-- **`optionally(transform)`**  
-  Wraps a given transform and returns its result if it is not `None`.
-  Otherwise returns the input path unchanged.
-  See below for example usage.
-* **`do(transforms)`**  
-  Accepts a series of transforms and applies them in the given order to the result of the previous one.
-  If any transform returns `None`, `do` short-circuits and also returns `None`.
-  This can be used to perform multiple renames in a row:
-  ```py
-  do(
-      # Move them
-      move_dir("Vorlesungsmaterial/Vorlesungsvideos/", "Vorlesung/Videos/"),
-      # Fix extensions (if they have any)
-      optionally(re_rename("(.*).m4v.mp4", "{1}.mp4")),
-      # Remove the 'dbs' prefix (if they have any)
-      optionally(re_rename("(?i)dbs-(.+)", "{1}")),
-  )
-  ```
-- **`attempt(transforms)`**  
-  Applies the passed transforms in the given order until it finds one that does not return `None`.
-  If it does not find any, it returns `None`.
-  This can be used to give a list of possible transformations and automatically pick the first one that fits:
-  ```py
-  attempt(
-      # Move all videos. If a video is passed in, this `re_move` will succeed
-      # and attempt short-circuits with the result.
-      re_move(r"Vorlesungsmaterial/.*/(.+?)\.mp4", "Vorlesung/Videos/{1}.mp4"),
-      # Move the whole folder to a nicer name - now without any mp4!
-      move_dir("Vorlesungsmaterial/", "Vorlesung/"),
-      # If we got another file, keep it.
-      keep,
-  )
-  ```
-
-All of these combinators are used in the provided example configs, if you want
-to see some more real-life usages.
-
-### A short, but commented example
-
-```py
-from pathlib import Path, PurePath
-from PFERD import Pferd
-from PFERD.ilias import IliasElementType
-from PFERD.transform import *
-
-# This filter will later be used by the ILIAS crawler to decide whether it
-# should crawl a directory (or directory-like structure).
-def filter_course(path: PurePath, type: IliasElementType) -> bool:
-    # Note that glob returns a Transform, which is a function from PurePath ->
-    # Optional[PurePath]. Because of this, we need to apply the result of
-    # 'glob' to our input path. The returned value will be truthy (a Path) if
-    # the transform succeeded, or `None` if it failed.
-
-    # We need to crawl the 'Tutorien' folder as it contains one that we want.
-    if glob("Tutorien/")(path):
-        return True
-    # If we found 'Tutorium 10', keep it!
-    if glob("Tutorien/Tutorium 10")(path):
-        return True
-    # Discard all other folders inside 'Tutorien'
-    if glob("Tutorien/*")(path):
-        return False
-
-    # All other dirs (including subdirs of 'Tutorium 10') should be searched :)
-    return True
-
-
-# This transform will later be used to rename a few files. It can also be used
-# to ignore some files.
-transform_course = attempt(
-    # We don't care about the other tuts and would instead prefer a cleaner
-    # directory structure.
-    move_dir("Tutorien/Tutorium 10/", "Tutorium/"),
-    # We don't want to modify any other files, so we're going to keep them
-    # exactly as they are.
-    keep
-)
-
-# Enable and configure the text output. Needs to be called before calling any
-# other PFERD methods.
-Pferd.enable_logging()
-# Create a Pferd instance rooted in the same directory as the script file. This
-# is not a test run, so files will be downloaded (default, can be omitted).
-pferd = Pferd(Path(__file__).parent, test_run=False)
-
-# Use the ilias_kit helper to synchronize an ILIAS course
-pferd.ilias_kit(
-    # The directory that all of the downloaded files should be placed in
-    "My_cool_course/",
-    # The course ID (found in the URL when on the course page in ILIAS)
-    "course id",
-    # A path to a cookie jar. If you synchronize multiple ILIAS courses,
-    # setting this to a common value requires you to only log in once.
-    cookies=Path("ilias_cookies.txt"),
-    # A transform can rename, move or filter out certain files
-    transform=transform_course,
-    # A crawl filter limits what paths the cralwer searches
-    dir_filter=filter_course,
-)
+```
+$ pferd --dump-config kit-ilias-web <url> <output_directory>
+```
+
+This will make PFERD write its current configuration to its default config file
+path. You can then run `pferd` without a command and it will execute the config
+file. Alternatively, you can use `--dump-config-to` and specify a path yourself.
+Using `--dump-config-to -` will print the configuration to stdout instead of a
+file, which is a good way to see what is actually going on when using a CLI
+command.
+
+Another good way to see what PFERD is doing is the `--explain` option. When
+enabled, PFERD explains in detail what it is doing and why. This can help with
+debugging your own config.
+
+If you don't want to run all crawlers from your config file, you can specify the
+crawlers you want to run with `--crawler` or `-C`, like this:
+
+```
+$ pferd -C crawler1 -C crawler2
+```
+
+## Advanced usage
+
+PFERD supports lots of different options. For example, you can configure PFERD
+to [use your system's keyring](CONFIG.md#the-keyring-authenticator) instead of
+prompting you for your username and password. PFERD also supports
+[transformation rules](CONFIG.md#transformation-rules) that let you rename or
+exclude certain files.
+
+For more details, see the comprehensive [config format documentation](CONFIG.md).
+
+## Example
+
+This example downloads a few courses from the KIT ILIAS with a common keyring
+authenticator. It reorganizes and ignores some files.
+
+```ini
+[DEFAULT]
+# All paths will be relative to this.
+# The crawler output directories will be <working_dir>/Foo and <working_dir>/Bar.
+working_dir = ~/stud
+# If files vanish from ILIAS the local files are not deleted, allowing us to
+# take a look at them before deleting them ourselves.
+on_conflict = no-delete
+
+[auth:ilias]
+type = keyring
+username = foo
+
+[crawl:Foo]
+type = kit-ilias-web
+auth = auth:ilias
+# Crawl a course by its ID (found as `ref_id=ID` in the URL)
+target = 1234567
+
+# Plaintext files are easier to read by other tools
+links = plaintext
+
+transform =
+  # Ignore unneeded folders
+  Online-Tests --> !
+  Vorlesungswerbung --> !
+
+  # Rename folders
+  Lehrbücher --> Vorlesung
+  # Note the ">>" arrow head which lets us apply further rules to files moved to "Übung"
+  Übungsunterlagen -->> Übung
+
+  # Move exercises to own folder. Rename them to "Blatt-XX.pdf" to make them sort properly
+  "Übung/(\d+). Übungsblatt.pdf" -re-> Blätter/Blatt-{i1:02}.pdf
+  # Move solutions to own folder. Rename them to "Blatt-XX-Lösung.pdf" to make them sort properly
+  "Übung/(\d+). Übungsblatt.*Musterlösung.pdf" -re-> Blätter/Blatt-{i1:02}-Lösung.pdf
+
+  # The course has nested folders with the same name - flatten them
+  "Übung/(.+?)/\\1" -re-> Übung/{g1}
+
+[crawl:Bar]
+type = kit-ilias-web
+auth = auth:ilias
+target = 1337420
 ```

example_config.py

@@ -1,131 +0,0 @@
-import argparse
-from pathlib import Path, PurePath
-
-from PFERD import Pferd
-from PFERD.ilias import IliasElementType
-from PFERD.transform import (attempt, do, glob, keep, move, move_dir,
-                             optionally, re_move, re_rename)
-
-tf_ss_2020_numerik = attempt(
-    re_move(r"Übungsblätter/(\d+)\. Übungsblatt/.*", "Blätter/Blatt_{1:0>2}.pdf"),
-    keep,
-)
-
-
-tf_ss_2020_db = attempt(
-    move_dir("Begrüßungsvideo/", "Vorlesung/Videos/"),
-    do(
-        move_dir("Vorlesungsmaterial/Vorlesungsvideos/", "Vorlesung/Videos/"),
-        optionally(re_rename("(.*).m4v.mp4", "{1}.mp4")),
-        optionally(re_rename("(?i)dbs-(.+)", "{1}")),
-    ),
-    move_dir("Vorlesungsmaterial/", "Vorlesung/"),
-    keep,
-)
-
-
-tf_ss_2020_rechnernetze = attempt(
-    re_move(r"Vorlesungsmaterial/.*/(.+?)\.mp4", "Vorlesung/Videos/{1}.mp4"),
-    move_dir("Vorlesungsmaterial/", "Vorlesung/"),
-    keep,
-)
-
-
-tf_ss_2020_sicherheit = attempt(
-    move_dir("Vorlesungsvideos/", "Vorlesung/Videos/"),
-    move_dir("Übungsvideos/", "Übung/Videos/"),
-    re_move(r"VL(.*)\.pdf", "Vorlesung/{1}.pdf"),
-    re_move(r"Übungsblatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf"),
-    move("Chiffrat.txt", "Blätter/Blatt_01_Chiffrat.txt"),
-    keep,
-)
-
-
-tf_ss_2020_pg = attempt(
-    move_dir("Vorlesungsaufzeichnungen/", "Vorlesung/Videos/"),
-    move_dir("Vorlesungsmaterial/", "Vorlesung/"),
-    re_move(r"Übungen/uebungsblatt(\d+).pdf", "Blätter/Blatt_{1:0>2}.pdf"),
-    keep,
-)
-
-
-def df_ss_2020_or1(path: PurePath, _type: IliasElementType) -> bool:
-    if glob("Tutorien/")(path):
-        return True
-    if glob("Tutorien/Tutorium 10, dienstags 15:45 Uhr/")(path):
-        return True
-    if glob("Tutorien/*")(path):
-        return False
-    return True
-
-
-tf_ss_2020_or1 = attempt(
-    move_dir("Vorlesung/Unbeschriebene Folien/", "Vorlesung/Folien/"),
-    move_dir("Video zur Organisation/", "Vorlesung/Videos/"),
-    keep,
-)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--test-run", action="store_true")
-    parser.add_argument("synchronizers", nargs="*")
-    args = parser.parse_args()
-
-    pferd = Pferd(Path(__file__).parent, test_run=args.test_run)
-    pferd.enable_logging()
-
-    if not args.synchronizers or "numerik" in args.synchronizers:
-        pferd.ilias_kit(
-            target="Numerik",
-            course_id="1083036",
-            transform=tf_ss_2020_numerik,
-            cookies="ilias_cookies.txt",
-        )
-
-    if not args.synchronizers or "db" in args.synchronizers:
-        pferd.ilias_kit(
-            target="DB",
-            course_id="1101554",
-            transform=tf_ss_2020_db,
-            cookies="ilias_cookies.txt",
-        )
-
-    if not args.synchronizers or "rechnernetze" in args.synchronizers:
-        pferd.ilias_kit(
-            target="Rechnernetze",
-            course_id="1099996",
-            transform=tf_ss_2020_rechnernetze,
-            cookies="ilias_cookies.txt",
-        )
-
-    if not args.synchronizers or "sicherheit" in args.synchronizers:
-        pferd.ilias_kit(
-            target="Sicherheit",
-            course_id="1101980",
-            transform=tf_ss_2020_sicherheit,
-            cookies="ilias_cookies.txt",
-        )
-
-    if not args.synchronizers or "pg" in args.synchronizers:
-        pferd.ilias_kit(
-            target="PG",
-            course_id="1106095",
-            transform=tf_ss_2020_pg,
-            cookies="ilias_cookies.txt",
-        )
-
-    if not args.synchronizers or "or1" in args.synchronizers:
-        pferd.ilias_kit(
-            target="OR1",
-            course_id="1105941",
-            dir_filter=df_ss_2020_or1,
-            transform=tf_ss_2020_or1,
-            cookies="ilias_cookies.txt",
-        )
-
-    # Prints a summary listing all new, modified or deleted files
-    pferd.print_summary()
-
-if __name__ == "__main__":
-    main()

example_config_personal_desktop.py

@@ -1,38 +0,0 @@
-"""
-This is a small config that just crawls the ILIAS Personal Desktop.
-It does not filter or rename anything, it just gobbles up everything it can find.
-
-Note that this still includes a test-run switch, so you can see what it *would* download.
-You can enable that with the "--test-run" command line switch,
-i. e. "python3 example_config_minimal.py --test-run".
-"""
-
-import argparse
-from pathlib import Path
-
-from PFERD import Pferd
-
-
-def main() -> None:
-    # Parse command line arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--test-run", action="store_true")
-    args = parser.parse_args()
-
-    # Create the Pferd helper instance
-    pferd = Pferd(Path(__file__).parent, test_run=args.test_run)
-    pferd.enable_logging()
-
-    # Synchronize the personal desktop into the "ILIAS" directory.
-    # It saves the cookies, so you only need to log in again when the ILIAS cookies expire.
-    pferd.ilias_kit_personal_desktop(
-        "ILIAS",
-        cookies="ilias_cookies.txt",
-    )
-
-    # Prints a summary listing all new, modified or deleted files
-    pferd.print_summary()
-
-
-if __name__ == "__main__":
-    main()

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1694499547,
+        "narHash": "sha256-R7xMz1Iia6JthWRHDn36s/E248WB1/je62ovC/dUVKI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e5f018cf150e29aac26c61dac0790ea023c46b24",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,41 @@
+{
+  description = "Tool for downloading course-related files from ILIAS";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+      # Helper function to generate an attrset '{ x86_64-linux = f "x86_64-linux"; ... }'.
+      forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+    in
+    {
+      packages = forAllSystems (system:
+        let pkgs = import nixpkgs { inherit system; };
+        in
+        rec {
+          default = pkgs.python3Packages.buildPythonApplication rec {
+            pname = "pferd";
+            # Performing black magic
+            # Don't worry, I sacrificed enough goats for the next few years
+            version = (pkgs.lib.importTOML ./PFERD/version.py).VERSION;
+            format = "pyproject";
+
+            src = ./.;
+
+            nativeBuildInputs = with pkgs.python3Packages; [
+              setuptools
+            ];
+
+            propagatedBuildInputs = with pkgs.python3Packages; [
+              aiohttp
+              beautifulsoup4
+              rich
+              keyring
+              certifi
+            ];
+          };
+        });
+    };
+}

mypy.ini

@@ -1,7 +1,11 @@
 [mypy]
+disallow_any_generics = True
 disallow_untyped_defs = True
 disallow_incomplete_defs = True
 no_implicit_optional = True
+warn_unused_ignores = True
+warn_unreachable = True
+show_error_context = True
 
-[mypy-rich.*,bs4]
+[mypy-rich.*,bs4,keyring]
 ignore_missing_imports = True

pferd.py

@@ -0,0 +1,6 @@
+# File used by pyinstaller to create the executable
+
+from PFERD.__main__ import main
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

scripts/build

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+pyinstaller --onefile pferd.py

scripts/bump-version

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+import argparse
+import time
+import re
+from subprocess import run
+
+
+def load_changelog():
+    with open("CHANGELOG.md") as f:
+        return list(f)
+
+
+def extract_changes(lines):
+    lines = iter(lines)
+    changes = []
+
+    # Find "Unreleased" section
+    for line in lines:
+        if line.strip() == "## Unreleased":
+            break
+    next(lines)
+
+    # Read all lines from that section
+    for line in lines:
+        if line.startswith("## "):
+            # Found the beginning of the next section
+            break
+        elif line.startswith("### "):
+            # Found a heading in the current section
+            # Remove "#" symbols so git doesn't interpret the line as a comment later
+            changes.append(line[4:])
+        else:
+            changes.append(line)
+
+    # Remove trailing empty lines
+    while changes and not changes[-1].strip():
+        changes.pop()
+
+    return changes
+
+
+def update_version(version):
+    with open("PFERD/version.py") as f:
+        text = f.read()
+
+    text = re.sub(r'VERSION = ".*"', f'VERSION = "{version}"', text)
+
+    with open("PFERD/version.py", "w") as f:
+        f.write(text)
+
+
+def update_changelog(lines, version, date):
+    lines = iter(lines)
+    new_lines = []
+
+    # Find "Unreleased" section
+    for line in lines:
+        new_lines.append(line)
+        if line.strip() == "## Unreleased":
+            break
+
+    # Add new heading below that
+    new_lines.append("\n")
+    new_lines.append(f"## {version} - {date}\n")
+
+    # Add remaining lines
+    for line in lines:
+        new_lines.append(line)
+
+    with open("CHANGELOG.md", "w") as f:
+        f.write("".join(new_lines))
+
+
+def commit_changes(version):
+    run(["git", "add", "CHANGELOG.md", "PFERD/version.py"])
+    run(["git", "commit", "-m", f"Bump version to {version}"])
+
+
+def create_tag(version, annotation):
+    run(["git", "tag", "-am", annotation, f"v{version}"])
+
+
+def fastforward_latest():
+    run(["git", "branch", "-f", "latest", "HEAD"])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version")
+    args = parser.parse_args()
+
+    version = args.version
+    date = time.strftime("%Y-%m-%d")
+    changelog = load_changelog()
+    changes = extract_changes(changelog)
+    annotation = f"Version {version} - {date}\n\n{''.join(changes)}"
+
+    update_version(version)
+    update_changelog(changelog, version, date)
+    commit_changes(version)
+    create_tag(version, annotation)
+    fastforward_latest()
+
+    print()
+    print("Now the only thing left is to publish the changes:")
+    print(f"  $ git push origin master latest v{version}")
+
+
+if __name__ == "__main__":
+    main()

scripts/check

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+mypy PFERD
+flake8 PFERD

scripts/format

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+autopep8 --recursive --in-place PFERD
+isort PFERD

scripts/setup

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Updating pip and setuptools because some older versions don't recognize the
+# project setup correctly
+if [[ $1 != '--no-pip' ]]; then
+    pip install --upgrade pip
+fi
+pip install --upgrade setuptools
+
+# Installing PFERD itself
+pip install --editable .
+
+# Installing tools and type hints
+pip install --upgrade mypy flake8 autopep8 isort pyinstaller
+pip install --upgrade types-chardet types-certifi

setup.cfg

@@ -0,0 +1,23 @@
+[metadata]
+name = PFERD
+version = attr: PFERD.version.VERSION
+
+[options]
+packages = find:
+python_requires = >=3.9
+install_requires =
+  aiohttp>=3.8.1
+  beautifulsoup4>=4.10.0
+  rich>=11.0.0
+  keyring>=23.5.0
+  certifi>=2021.10.8
+
+[options.entry_points]
+console_scripts =
+  pferd = PFERD.__main__:main
+
+[flake8]
+max_line_length = 110
+
+[isort]
+line_length = 110

setup.py

@@ -1,16 +0,0 @@
-from setuptools import find_packages, setup
-
-setup(
-    name="PFERD",
-    version="2.4.0",
-    packages=find_packages(),
-    install_requires=[
-        "requests>=2.21.0",
-        "beautifulsoup4>=4.7.1",
-        "rich>=2.1.0"
-    ],
-)
-
-# When updating the version, also:
-# - update the README.md installation instructions
-# - set a tag on the update commit

sync_url.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-
-"""
-A simple script to download a course by name from ILIAS.
-"""
-
-import argparse
-from pathlib import Path
-from urllib.parse import urlparse
-
-from PFERD import Pferd
-from PFERD.cookie_jar import CookieJar
-from PFERD.ilias import (IliasCrawler, IliasElementType,
-                         KitShibbolethAuthenticator)
-from PFERD.utils import to_path
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--test-run", action="store_true")
-    parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in")
-    parser.add_argument('-f', '--no-videos', nargs='?', default=None, help="Don't download videos")
-    parser.add_argument('url', help="URL to the course page")
-    parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into")
-    args = parser.parse_args()
-
-    url = urlparse(args.url)
-
-    cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None)
-    session = cookie_jar.create_session()
-    authenticator = KitShibbolethAuthenticator()
-    crawler = IliasCrawler(url.scheme + '://' + url.netloc, session,
-                           authenticator, lambda x, y: True)
-
-    cookie_jar.load_cookies()
-
-    if args.folder is not None:
-        folder = args.folder
-        # Initialize pferd at the *parent of the passed folder*
-        # This is needed so Pferd's internal protections against escaping the working directory
-        # do not trigger (e.g. if somebody names a file in ILIAS '../../bad thing.txt')
-        pferd = Pferd(Path(Path(__file__).parent, folder).parent, test_run=args.test_run)
-    else:
-        # fetch course name from ilias
-        folder = crawler.find_element_name(args.url)
-        cookie_jar.save_cookies()
-
-        # Initialize pferd at the location of the script
-        pferd = Pferd(Path(__file__).parent, test_run=args.test_run)
-
-    def dir_filter(_: Path, element: IliasElementType) -> bool:
-        if args.no_videos:
-            return element not in [IliasElementType.VIDEO_FILE, IliasElementType.VIDEO_FOLDER]
-        return True
-
-    pferd.enable_logging()
-    # fetch
-    pferd.ilias_kit_folder(
-        target=folder,
-        full_url=args.url,
-        cookies=args.cookies,
-        dir_filter=dir_filter
-    )
-
-
-if __name__ == "__main__":
-    main()