Merge 77c1f1516c into e41a22149e

If set to `no`, PFERD won't print status or report messages for not deleted files

.github/workflows/build-and-release.yml

@@ -14,12 +14,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8"]
+        python: ["3.9"]
     steps:
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 
@@ -45,7 +45,7 @@ jobs:
         run: mv dist/pferd* dist/pferd-${{ matrix.os }}
 
       - name: Upload binary
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: Binaries
           path: dist/pferd-${{ matrix.os }}
@@ -57,7 +57,7 @@ jobs:
     steps:
 
       - name: Download binaries
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: Binaries
 

CHANGELOG.md

@@ -22,6 +22,151 @@ ambiguous situations.
 
 ## Unreleased
 
+### Fixed
+- Crawling of courses with the timeline view as the default tab
+- Crawling of file and custom opencast cards
+- Crawling of button cards without descriptions
+- Abort crawling when encountering an unexpected ilias root page redirect
+
+### Added
+- `no-delete-prompt-override` conflict resolution strategy
+- support for ILIAS learning modules
+- `show_not_deleted` option to stop printing the "Not Deleted" status or report
+  message. This combines nicely with the `no-delete-prompt-override` strategy,
+  causing PFERD to mostly ignore local-only files.
+
+## 3.4.3 - 2022-11-29
+
+### Added
+- Missing documentation for `forums` option
+
+### Changed
+- Clear up error message shown when multiple paths are found to an element
+
+### Fixed
+- IPD crawler unnecessarily appending trailing slashes
+- Crawling opencast when ILIAS is set to English
+
+## 3.4.2 - 2022-10-26
+
+### Added
+- Recognize and crawl content pages in cards
+- Recognize and ignore surveys
+
+### Fixed
+- Forum crawling crashing when a thread has no messages at all
+- Forum crawling crashing when a forum has no threads at all
+- Ilias login failing in some cases
+- Crawling of paginated future meetings
+- IPD crawler handling of URLs without trailing slash
+
+## 3.4.1 - 2022-08-17
+
+### Added
+- Download of page descriptions
+- Forum download support
+- `pass` authenticator
+
+### Changed
+- Add `cpp` extension to default `link_regex` of IPD crawler
+- Mention hrefs in IPD crawler's `--explain` output for users of `link_regex` option
+- Simplify default IPD crawler `link_regex`
+
+### Fixed
+- IPD crawler crashes on some sites
+- Meeting name normalization for yesterday, today and tomorrow
+- Crawling of meeting file previews
+- Login with new login button html layout
+- Descriptions for courses are now placed in the correct subfolder when
+  downloading the whole desktop
+
+## 3.4.0 - 2022-05-01
+
+### Added
+- Message when Shibboleth entitlements need to be manually reviewed
+- Links to unofficial packages and repology in the readme
+
+### Changed
+- Increase minimum supported Python version to 3.9
+- Support video listings with more columns
+- Use UTF-8 when reading/writing the config file
+
+### Fixed
+- Crash during authentication when the Shibboleth session is still valid
+
+## 3.3.1 - 2022-01-15
+
+### Fixed
+- ILIAS login
+- Local video cache if `windows_paths` is enabled
+
+## 3.3.0 - 2022-01-09
+
+### Added
+- A KIT IPD crawler
+- Support for ILIAS cards
+- (Rudimentary) support for content pages
+- Support for multi-stream videos
+- Support for ILIAS 7
+
+### Removed
+- [Interpolation](https://docs.python.org/3/library/configparser.html#interpolation-of-values) in config file
+
+### Fixed
+- Crawling of recursive courses
+- Crawling files directly placed on the personal desktop
+- Ignore timestamps at the unix epoch as they crash on windows
+
+## 3.2.0 - 2021-08-04
+
+### Added
+- `--skip` command line option
+- Support for ILIAS booking objects
+
+### Changed
+- Using multiple path segments on left side of `-name->` now results in an
+  error. This was already forbidden by the documentation but silently accepted
+  by PFERD.
+- More consistent path printing in some `--explain` messages
+
+### Fixed
+- Nondeterministic name deduplication due to ILIAS reordering elements
+- More exceptions are handled properly
+
+## 3.1.0 - 2021-06-13
+
+If your config file doesn't do weird things with transforms, it should continue
+to work. If your `-re->` arrows behave weirdly, try replacing them with
+`-exact-re->` arrows. If you're on Windows, you might need to switch from `\`
+path separators to `/` in your regex rules.
+
+### Added
+- `skip` option for crawlers
+- Rules with `>>` instead of `>` as arrow head
+- `-exact-re->` arrow (behaves like `-re->` did previously)
+
+### Changed
+- The `-re->` arrow can now rename directories (like `-->`)
+- Use `/` instead of `\` as path separator for (regex) rules on Windows
+- Use the label to the left for exercises instead of the button name to
+  determine the folder name
+
+### Fixed
+- Video pagination handling in ILIAS crawler
+
+## 3.0.1 - 2021-06-01
+
+### Added
+- `credential-file` authenticator
+- `--credential-file` option for `kit-ilias-web` command
+- Warning if using concurrent tasks with `kit-ilias-web`
+
+### Changed
+- Cookies are now stored in a text-based format
+
+### Fixed
+- Date parsing now also works correctly in non-group exercises
+
 ## 3.0.0 - 2021-05-31
 
 ### Added

CONFIG.md

@@ -4,11 +4,11 @@ A config file consists of sections. A section begins with a `[section]` header,
 which is followed by a list of `key = value` pairs. Comments must be on their
 own line and start with `#`. Multiline values must be indented beyond their key.
 Boolean values can be `yes` or `no`. For more details and some examples on the
-format, see the [configparser documentation][1] ([basic interpolation][2] is
+format, see the [configparser documentation][1] ([interpolation][2] is
-enabled).
+disabled).
 
 [1]: <https://docs.python.org/3/library/configparser.html#supported-ini-file-structure> "Supported INI File Structure"
-[2]: <https://docs.python.org/3/library/configparser.html#configparser.BasicInterpolation> "BasicInterpolation"
+[2]: <https://docs.python.org/3/library/configparser.html#interpolation-of-values> "Interpolation of values"
 
 ## The `DEFAULT` section
 
@@ -26,6 +26,9 @@ default values for the other sections.
   `Added ...`) while running a crawler. (Default: `yes`)
 - `report`: Whether PFERD should print a report of added, changed and deleted
    local files for all crawlers before exiting. (Default: `yes`)
+- `show_not_deleted`: Whether PFERD should print messages in status and report
+   when a local-only file wasn't deleted. Combines nicely with the
+   `no-delete-prompt-override` conflict resolution strategy.
 - `share_cookies`: Whether crawlers should share cookies where applicable. For
   example, some crawlers share cookies if they crawl the same website using the
   same account. (Default: `yes`)
@@ -49,8 +52,11 @@ see the type's [documentation](#crawler-types) below. The following options are
 common to all crawlers:
 
 - `type`: The available types are specified in [this section](#crawler-types).
+- `skip`: Whether the crawler should be skipped during normal execution. The
+  crawler can still be executed manually using the `--crawler` or `-C` flags.
+  (Default: `no`)
 - `output_dir`: The directory the crawler synchronizes files to. A crawler will
-  never place any files outside of this directory. (Default: the crawler's name)
+  never place any files outside this directory. (Default: the crawler's name)
 - `redownload`: When to download a file that is already present locally.
   (Default: `never-smart`)
     - `never`: If a file is present locally, it is not downloaded again.
@@ -72,6 +78,9 @@ common to all crawlers:
       using `prompt` and always choosing "yes".
     - `no-delete`: Never delete local files, but overwrite local files if the
       remote file is different.
+    - `no-delete-prompt-overwrite`: Never delete local files, but prompt to
+      overwrite local files if the remote file is different. Combines nicely
+      with the `show_not_deleted` option.
 - `transform`: Rules for renaming and excluding certain files and directories.
   For more details, see [this section](#transformation-rules). (Default: empty)
 - `tasks`: The maximum number of concurrent tasks (such as crawling or
@@ -83,6 +92,9 @@ common to all crawlers:
   load for the crawl target. (Default: `0.0`)
 - `windows_paths`: Whether PFERD should find alternative names for paths that
   are invalid on Windows. (Default: `yes` on Windows, `no` otherwise)
+- `aliases`: List of strings that are considered as an alias when invoking with
+  the `--crawler` or `-C` flag. If there is more than one crawl section with
+  the same aliases all are selected. Thereby, you can group different crawlers.
 
 Some crawlers may also require credentials for authentication. To configure how
 the crawler obtains its credentials, the `auth` option is used. It is set to the
@@ -97,6 +109,7 @@ username = foo
 password = bar
 
 [crawl:something]
+aliases = [sth, some]
 type = some-complex-crawler
 auth = auth:example
 on_conflict = no-delete
@@ -133,6 +146,18 @@ crawler simulate a slower, network-based crawler.
   requests. (Default: `0.0`)
 - `download_speed`: Download speed (in bytes per second) to simulate. (Optional)
 
+### The `kit-ipd` crawler
+
+This crawler crawls a KIT-IPD page by url. The root page can be crawled from
+outside the KIT network so you will be informed about any new/deleted files,
+but downloading files requires you to be within. Adding a show delay between
+requests is likely a good idea.
+
+- `target`: URL to a KIT-IPD page
+- `link_regex`: A regex that is matched against the `href` part of links. If it
+  matches, the given link is downloaded as a file. This is used to extract
+  files from KIT-IPD pages. (Default: `^.*?[^/]+\.(pdf|zip|c|cpp|java)$`)
+
 ### The `kit-ilias-web` crawler
 
 This crawler crawls the KIT ILIAS instance.
@@ -166,6 +191,7 @@ script once per day should be fine.
   redirect to the actual URL. Set to a negative value to disable the automatic
   redirect. (Default: `-1`)
 - `videos`: Whether to download videos. (Default: `no`)
+- `forums`: Whether to download forum threads. (Default: `no`)
 - `http_timeout`: The timeout (in seconds) for all HTTP requests. (Default:
   `20.0`)
 
@@ -180,6 +206,22 @@ via the terminal.
 - `username`: The username. (Optional)
 - `password`: The password. (Optional)
 
+### The `credential-file` authenticator
+
+This authenticator reads a username and a password from a credential file.
+
+- `path`: Path to the credential file. (Required)
+
+The credential file has exactly two lines (trailing newline optional). The first
+line starts with `username=` and contains the username, the second line starts
+with `password=` and contains the password. The username and password may
+contain any characters except a line break.
+
+```
+username=AzureDiamond
+password=hunter2
+```
+
 ### The `keyring` authenticator
 
 This authenticator uses the system keyring to store passwords. The username can
@@ -192,6 +234,23 @@ is stored in the keyring.
 - `keyring_name`: The service name PFERD uses for storing credentials. (Default:
   `PFERD`)
 
+### The `pass` authenticator
+
+This authenticator queries the [`pass` password manager][3] for a username and
+password. It tries to be mostly compatible with [browserpass][4] and
+[passff][5], so see those links for an overview of the format. If PFERD fails
+to load your password, you can use the `--explain` flag to see why.
+
+- `passname`: The name of the password to use (Required)
+- `username_prefixes`: A comma-separated list of username line prefixes
+  (Default: `login,username,user`)
+- `password_prefixes`: A comma-separated list of password line prefixes
+  (Default: `password,pass,secret`)
+
+[3]: <https://www.passwordstore.org/> "Pass: The Standard Unix Password Manager"
+[4]: <https://github.com/browserpass/browserpass-extension#organizing-password-store> "Organizing password store"
+[5]: <https://github.com/passff/passff#multi-line-format> "Multi-line format"
+
 ### The `tfa` authenticator
 
 This authenticator prompts the user on the console for a two-factor
@@ -203,58 +262,89 @@ This authenticator does not support usernames.
 Transformation rules are rules for renaming and excluding files and directories.
 They are specified line-by-line in a crawler's `transform` option. When a
 crawler needs to apply a rule to a path, it goes through this list top-to-bottom
-and choose the first matching rule.
+and applies the first matching rule.
 
 To see this process in action, you can use the `--debug-transforms` or flag or
 the `--explain` flag.
 
-Each line has the format `SOURCE ARROW TARGET` where `TARGET` is optional.
+Each rule has the format `SOURCE ARROW TARGET` (e. g. `foo/bar --> foo/baz`).
-`SOURCE` is either a normal path without spaces (e. g. `foo/bar`), or a string
+The arrow specifies how the source and target are interpreted. The different
-literal delimited by `"` or `'` (e. g. `"foo\" bar/baz"`). Python's string
+kinds of arrows are documented below.
-escape syntax is supported. Trailing slashes are ignored. `TARGET` can be
-formatted like `SOURCE`, but it can also be a single exclamation mark without
-quotes (`!`). `ARROW` is one of `-->`, `-name->`, `-exact->`, `-re->` and
-`-name-re->`
 
-If a rule's target is `!`, this means that when the rule matches on a path, the
+`SOURCE` and `TARGET` are either a bunch of characters without spaces (e. g.
-corresponding file or directory is ignored. If a rule's target is missing, the
+`foo/bar`) or string literals (e. g, `"foo/b a r"`). The former syntax has no
-path is matched but not modified.
+concept of escaping characters, so the backslash is just another character. The
+string literals however support Python's escape syntax (e. g.
+`"foo\\bar\tbaz"`). This also means that in string literals, backslashes must be
+escaped.
+
+`TARGET` can additionally be a single exclamation mark `!` (*not* `"!"`). When a
+rule with a `!` as target matches a path, the corresponding file or directory is
+ignored by the crawler instead of renamed.
+
+`TARGET` can also be omitted entirely. When a rule without target matches a
+path, the path is returned unmodified. This is useful to prevent rules further
+down from matching instead.
+
+Each arrow's behaviour can be modified slightly by changing the arrow's head
+from `>` to `>>`. When a rule with a `>>` arrow head matches a path, it doesn't
+return immediately like a normal arrow. Instead, it replaces the current path
+with its output and continues on to the next rule. In effect, this means that
+multiple rules can be applied sequentially.
 
 ### The `-->` arrow
 
-The `-->` arrow is a basic renaming operation. If a path begins with `SOURCE`,
+The `-->` arrow is a basic renaming operation for files and directories. If a
-that part of the path is replaced with `TARGET`. This means that the rule
+path matches `SOURCE`, it is renamed to `TARGET`.
-`foo/bar --> baz` would convert `foo/bar` into `baz`, but also `foo/bar/xyz`
+
-into `baz/xyz`. The rule `foo --> !` would ignore a directory named `foo` as
+Example: `foo/bar --> baz`
-well as all its contents.
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Converts `foo/bar` into `baz`
+- Converts `foo/bar/wargl` into `baz/wargl`
+
+Example: `foo/bar --> !`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Ignores `foo/bar` and any of its children
 
 ### The `-name->` arrow
 
 The `-name->` arrow lets you rename files and directories by their name,
 regardless of where they appear in the file tree. Because of this, its `SOURCE`
 must not contain multiple path segments, only a single name. This restriction
-does not apply to its `TARGET`. The `-name->` arrow is not applied recursively
+does not apply to its `TARGET`.
-to its own output to prevent infinite loops.
 
-For example, the rule `foo -name-> bar/baz` would convert `a/foo` into
+Example: `foo -name-> bar/baz`
-`a/bar/baz` and `a/foo/b/c/foo` into `a/bar/baz/b/c/bar/baz`. The rule `foo
+- Doesn't match `a/foobar/b` or `x/Foo/y/z`
--name-> !` would ignore all directories and files named `foo`.
+- Converts `hello/foo` into `hello/bar/baz`
+- Converts `foo/world` into `bar/baz/world`
+- Converts `a/foo/b/c/foo` into `a/bar/baz/b/c/bar/baz`
+
+Example: `foo -name-> !`
+- Doesn't match `a/foobar/b` or `x/Foo/y/z`
+- Ignores any path containing a segment `foo`
 
 ### The `-exact->` arrow
 
-The `-exact->` arrow requires the path to match `SOURCE` exactly. This means
+The `-exact->` arrow requires the path to match `SOURCE` exactly. The examples
-that the rule `foo/bar -exact-> baz` would still convert `foo/bar` into `baz`,
+below show why this is useful.
-but `foo/bar/xyz` would be unaffected. Also, `foo -exact-> !` would only ignore
+
-`foo`, but not its contents (if it has any). The examples below show why this is
+Example: `foo/bar -exact-> baz`
-useful.
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Converts `foo/bar` into `baz`
+- Doesn't match `foo/bar/wargl`
+
+Example: `foo/bar -exact-> !`
+- Doesn't match `foo`, `a/foo/bar` or `foo/baz`
+- Ignores only `foo/bar`, not its children
 
 ### The `-re->` arrow
 
-The `-re->` arrow uses regular expressions. `SOURCE` is a regular expression
+The `-re->` arrow is like the `-->` arrow but with regular expressions. `SOURCE`
-that must match the entire path. If this is the case, then the capturing groups
+is a regular expression and `TARGET` an f-string based template. If a path
-are available in `TARGET` for formatting.
+matches `SOURCE`, the output path is created using `TARGET` as template.
+`SOURCE` is automatically anchored.
 
-`TARGET` uses Python's [format string syntax][3]. The *n*-th capturing group can
+`TARGET` uses Python's [format string syntax][6]. The *n*-th capturing group can
 be referred to as `{g<n>}` (e.g. `{g3}`). `{g0}` refers to the original path.
 If capturing group *n*'s contents are a valid integer, the integer value is
 available as `{i<n>}` (e.g. `{i3}`). If capturing group *n*'s contents are a
@@ -269,18 +359,37 @@ can use `{i3:05}`.
 PFERD even allows you to write entire expressions inside the curly braces, for
 example `{g2.lower()}` or `{g3.replace(' ', '_')}`.
 
-[3]: <https://docs.python.org/3/library/string.html#format-string-syntax> "Format String Syntax"
+Example: `f(oo+)/be?ar -re-> B{g1.upper()}H/fear`
+- Doesn't match `a/foo/bar`, `foo/abc/bar`, `afoo/bar` or `foo/bars`
+- Converts `foo/bar` into `BOOH/fear`
+- Converts `fooooo/bear` into `BOOOOOH/fear`
+- Converts `foo/bar/baz` into `BOOH/fear/baz`
+
+[6]: <https://docs.python.org/3/library/string.html#format-string-syntax> "Format String Syntax"
 
 ### The `-name-re->` arrow
 
 The `-name-re>` arrow is like a combination of the `-name->` and `-re->` arrows.
-Instead of the `SOURCE` being the name of a directory or file, it's a regex that
-is matched against the names of directories and files. `TARGET` works like the
-`-re->` arrow's target.
 
-For example, the arrow `(.*)\.jpeg -name-re-> {g1}.jpg` will rename all `.jpeg`
+Example: `(.*)\.jpeg -name-re-> {g1}.jpg`
-extensions into `.jpg`. The arrow `\..+ -name-re-> !` will ignore all files and
+- Doesn't match `foo/bar.png`, `baz.JPEG` or `hello,jpeg`
-directories starting with `.`.
+- Converts `foo/bar.jpeg` into `foo/bar.jpg`
+- Converts `foo.jpeg/bar/baz.jpeg` into `foo.jpg/bar/baz.jpg`
+
+Example: `\..+ -name-re-> !`
+- Doesn't match `.`, `test`, `a.b`
+- Ignores all files and directories starting with `.`.
+
+### The `-exact-re->` arrow
+
+The `-exact-re>` arrow is like a combination of the `-exact->` and `-re->`
+arrows.
+
+Example: `f(oo+)/be?ar -exactre-> B{g1.upper()}H/fear`
+- Doesn't match `a/foo/bar`, `foo/abc/bar`, `afoo/bar` or `foo/bars`
+- Converts `foo/bar` into `BOOH/fear`
+- Converts `fooooo/bear` into `BOOOOOH/fear`
+- Doesn't match `foo/bar/baz`
 
 ### Example: Tutorials
 
@@ -307,8 +416,7 @@ tutorials --> !
 The second rule is required for many crawlers since they use the rules to decide
 which directories to crawl. If it was missing when the crawler looks at
 `tutorials/`, the third rule would match. This means the crawler would not crawl
-the `tutorials/` directory and thus not discover that `tutorials/tut02/`
+the `tutorials/` directory and thus not discover that `tutorials/tut02/` exists.
-existed.
 
 Since the second rule is only relevant for crawling, the `TARGET` is left out.
 
@@ -333,9 +441,9 @@ To do this, you can use the most powerful of arrows: The regex arrow.
 
 Note the escaped backslashes on the `SOURCE` side.
 
-### Example: Crawl a python project
+### Example: Crawl a Python project
 
-You are crawling a python project and want to ignore all hidden files (files
+You are crawling a Python project and want to ignore all hidden files (files
 whose name starts with a `.`), all `__pycache__` directories and all markdown
 files (for some weird reason).
 
@@ -355,11 +463,21 @@ README.md
 ...
 ```
 
-For this task, the name arrows can be used. They are variants of the normal
+For this task, the name arrows can be used.
-arrows that only look at the file name instead of the entire path.
 
 ```
 \..*        -name-re-> !
 __pycache__ -name->    !
 .*\.md      -name-re-> !
 ```
+
+### Example: Clean up names
+
+You want to convert all paths into lowercase and replace spaces with underscores
+before applying any rules. This can be achieved using the `>>` arrow heads.
+
+```
+(.*) -re->> "{g1.lower().replace(' ', '_')}"
+
+<other rules go here>
+```

LICENSE

@@ -1,4 +1,6 @@
-Copyright 2019-2020 Garmelon, I-Al-Istannen, danstooamerican, pavelzw, TheChristophe, Scriptim
+Copyright 2019-2021 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
+                    TheChristophe, Scriptim, thelukasprobst, Toorero,
+                    Mr-Pine
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

PFERD/__main__.py

@@ -5,7 +5,8 @@ import os
 import sys
 from pathlib import Path
 
-from .cli import PARSER, load_default_section
+from .auth import AuthLoadError
+from .cli import PARSER, ParserLoadError, load_default_section
 from .config import Config, ConfigDumpError, ConfigLoadError, ConfigOptionError
 from .logging import log
 from .pferd import Pferd, PferdLoadError
@@ -14,7 +15,7 @@ from .transformer import RuleParseError
 
 def load_config_parser(args: argparse.Namespace) -> configparser.ConfigParser:
     log.explain_topic("Loading config")
-    parser = configparser.ConfigParser()
+    parser = configparser.ConfigParser(interpolation=None)
 
     if args.command is None:
         log.explain("No CLI command specified, loading config from file")
@@ -36,6 +37,9 @@ def load_config(args: argparse.Namespace) -> Config:
         log.error(str(e))
         log.error_contd(e.reason)
         sys.exit(1)
+    except ParserLoadError as e:
+        log.error(str(e))
+        sys.exit(1)
 
 
 def configure_logging_from_args(args: argparse.Namespace) -> None:
@@ -43,6 +47,8 @@ def configure_logging_from_args(args: argparse.Namespace) -> None:
         log.output_explain = args.explain
     if args.status is not None:
         log.output_status = args.status
+    if args.show_not_deleted is not None:
+        log.output_not_deleted = args.show_not_deleted
     if args.report is not None:
         log.output_report = args.report
 
@@ -68,6 +74,8 @@ def configure_logging_from_config(args: argparse.Namespace, config: Config) -> N
             log.output_status = config.default_section.status()
         if args.report is None:
             log.output_report = config.default_section.report()
+        if args.show_not_deleted is None:
+            log.output_not_deleted = config.default_section.show_not_deleted()
     except ConfigOptionError as e:
         log.error(str(e))
         sys.exit(1)
@@ -112,7 +120,7 @@ def main() -> None:
         sys.exit()
 
     try:
-        pferd = Pferd(config, args.crawler)
+        pferd = Pferd(config, args.crawler, args.skip)
     except PferdLoadError as e:
         log.unlock()
         log.error(str(e))
@@ -131,7 +139,7 @@ def main() -> None:
             loop.close()
         else:
             asyncio.run(pferd.run(args.debug_transforms))
-    except ConfigOptionError as e:
+    except (ConfigOptionError, AuthLoadError) as e:
         log.unlock()
         log.error(str(e))
         sys.exit(1)
@@ -143,7 +151,6 @@ def main() -> None:
         log.unlock()
         log.explain_topic("Interrupted, exiting immediately")
         log.explain("Open files and connections are left for the OS to clean up")
-        log.explain("Temporary files are not cleaned up")
         pferd.print_report()
         # TODO Clean up tmp files
         # And when those files *do* actually get cleaned up properly,
@@ -156,3 +163,7 @@ def main() -> None:
         sys.exit(1)
     else:
         pferd.print_report()
+
+
+if __name__ == "__main__":
+    main()

PFERD/auth/__init__.py

@@ -2,8 +2,10 @@ from configparser import SectionProxy
 from typing import Callable, Dict
 
 from ..config import Config
-from .authenticator import Authenticator, AuthError, AuthSection  # noqa: F401
+from .authenticator import Authenticator, AuthError, AuthLoadError, AuthSection  # noqa: F401
+from .credential_file import CredentialFileAuthenticator, CredentialFileAuthSection
 from .keyring import KeyringAuthenticator, KeyringAuthSection
+from .pass_ import PassAuthenticator, PassAuthSection
 from .simple import SimpleAuthenticator, SimpleAuthSection
 from .tfa import TfaAuthenticator
 
@@ -14,10 +16,14 @@ AuthConstructor = Callable[[
 ], Authenticator]
 
 AUTHENTICATORS: Dict[str, AuthConstructor] = {
+    "credential-file": lambda n, s, c:
+        CredentialFileAuthenticator(n, CredentialFileAuthSection(s), c),
+    "keyring": lambda n, s, c:
+        KeyringAuthenticator(n, KeyringAuthSection(s)),
+    "pass": lambda n, s, c:
+        PassAuthenticator(n, PassAuthSection(s)),
     "simple": lambda n, s, c:
         SimpleAuthenticator(n, SimpleAuthSection(s)),
     "tfa": lambda n, s, c:
         TfaAuthenticator(n),
-    "keyring": lambda n, s, c:
-        KeyringAuthenticator(n, KeyringAuthSection(s))
 }

PFERD/auth/authenticator.py

@@ -13,14 +13,15 @@ class AuthError(Exception):
 
 
 class AuthSection(Section):
-    pass
+    def type(self) -> str:
+        value = self.s.get("type")
+        if value is None:
+            self.missing_value("type")
+        return value
 
 
 class Authenticator(ABC):
-    def __init__(
+    def __init__(self, name: str) -> None:
-            self,
-            name: str
-    ) -> None:
         """
         Initialize an authenticator from its name and its section in the config
         file.

PFERD/auth/credential_file.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+from typing import Tuple
+
+from ..config import Config
+from ..utils import fmt_real_path
+from .authenticator import Authenticator, AuthLoadError, AuthSection
+
+
+class CredentialFileAuthSection(AuthSection):
+    def path(self) -> Path:
+        value = self.s.get("path")
+        if value is None:
+            self.missing_value("path")
+        return Path(value)
+
+
+class CredentialFileAuthenticator(Authenticator):
+    def __init__(self, name: str, section: CredentialFileAuthSection, config: Config) -> None:
+        super().__init__(name)
+
+        path = config.default_section.working_dir() / section.path()
+        try:
+            with open(path, encoding="utf-8") as f:
+                lines = list(f)
+        except UnicodeDecodeError:
+            raise AuthLoadError(f"Credential file at {fmt_real_path(path)} is not encoded using UTF-8")
+        except OSError as e:
+            raise AuthLoadError(f"No credential file at {fmt_real_path(path)}") from e
+
+        if len(lines) != 2:
+            raise AuthLoadError("Credential file must be two lines long")
+        [uline, pline] = lines
+        uline = uline[:-1]  # Remove trailing newline
+        if pline.endswith("\n"):
+            pline = pline[:-1]
+
+        if not uline.startswith("username="):
+            raise AuthLoadError("First line must start with 'username='")
+        if not pline.startswith("password="):
+            raise AuthLoadError("Second line must start with 'password='")
+
+        self._username = uline[9:]
+        self._password = pline[9:]
+
+    async def credentials(self) -> Tuple[str, str]:
+        return self._username, self._password

PFERD/auth/keyring.py

@@ -18,11 +18,7 @@ class KeyringAuthSection(AuthSection):
 
 class KeyringAuthenticator(Authenticator):
 
-    def __init__(
+    def __init__(self, name: str, section: KeyringAuthSection) -> None:
-            self,
-            name: str,
-            section: KeyringAuthSection,
-    ) -> None:
         super().__init__(name)
 
         self._username = section.username()

PFERD/auth/pass_.py

@@ -0,0 +1,98 @@
+import re
+import subprocess
+from typing import List, Tuple
+
+from ..logging import log
+from .authenticator import Authenticator, AuthError, AuthSection
+
+
+class PassAuthSection(AuthSection):
+    def passname(self) -> str:
+        if (value := self.s.get("passname")) is None:
+            self.missing_value("passname")
+        return value
+
+    def username_prefixes(self) -> List[str]:
+        value = self.s.get("username_prefixes", "login,username,user")
+        return [prefix.lower() for prefix in value.split(",")]
+
+    def password_prefixes(self) -> List[str]:
+        value = self.s.get("password_prefixes", "password,pass,secret")
+        return [prefix.lower() for prefix in value.split(",")]
+
+
+class PassAuthenticator(Authenticator):
+    PREFIXED_LINE_RE = r"([a-zA-Z]+):\s?(.*)"  # to be used with fullmatch
+
+    def __init__(self, name: str, section: PassAuthSection) -> None:
+        super().__init__(name)
+
+        self._passname = section.passname()
+        self._username_prefixes = section.username_prefixes()
+        self._password_prefixes = section.password_prefixes()
+
+    async def credentials(self) -> Tuple[str, str]:
+        log.explain_topic("Obtaining credentials from pass")
+
+        try:
+            log.explain(f"Calling 'pass show {self._passname}'")
+            result = subprocess.check_output(["pass", "show", self._passname], text=True)
+        except subprocess.CalledProcessError as e:
+            raise AuthError(f"Failed to get password info from {self._passname}: {e}")
+
+        prefixed = {}
+        unprefixed = []
+        for line in result.strip().splitlines():
+            if match := re.fullmatch(self.PREFIXED_LINE_RE, line):
+                prefix = match.group(1).lower()
+                value = match.group(2)
+                log.explain(f"Found prefixed line {line!r} with prefix {prefix!r}, value {value!r}")
+                if prefix in prefixed:
+                    raise AuthError(f"Prefix {prefix} specified multiple times")
+                prefixed[prefix] = value
+            else:
+                log.explain(f"Found unprefixed line {line!r}")
+                unprefixed.append(line)
+
+        username = None
+        for prefix in self._username_prefixes:
+            log.explain(f"Looking for username at prefix {prefix!r}")
+            if prefix in prefixed:
+                username = prefixed[prefix]
+                log.explain(f"Found username {username!r}")
+                break
+
+        password = None
+        for prefix in self._password_prefixes:
+            log.explain(f"Looking for password at prefix {prefix!r}")
+            if prefix in prefixed:
+                password = prefixed[prefix]
+                log.explain(f"Found password {password!r}")
+                break
+
+        if password is None and username is None:
+            log.explain("No username and password found so far")
+            log.explain("Using first unprefixed line as password")
+            log.explain("Using second unprefixed line as username")
+        elif password is None:
+            log.explain("No password found so far")
+            log.explain("Using first unprefixed line as password")
+        elif username is None:
+            log.explain("No username found so far")
+            log.explain("Using first unprefixed line as username")
+
+        if password is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Password could not be determined")
+            password = unprefixed.pop(0)
+            log.explain(f"Found password {password!r}")
+
+        if username is None:
+            if not unprefixed:
+                log.explain("Not enough unprefixed lines left")
+                raise AuthError("Username could not be determined")
+            username = unprefixed.pop(0)
+            log.explain(f"Found username {username!r}")
+
+        return username, password

PFERD/auth/simple.py

@@ -14,11 +14,7 @@ class SimpleAuthSection(AuthSection):
 
 
 class SimpleAuthenticator(Authenticator):
-    def __init__(
+    def __init__(self, name: str, section: SimpleAuthSection) -> None:
-            self,
-            name: str,
-            section: SimpleAuthSection,
-    ) -> None:
         super().__init__(name)
 
         self._username = section.username()

PFERD/auth/tfa.py

@@ -6,10 +6,7 @@ from .authenticator import Authenticator, AuthError
 
 
 class TfaAuthenticator(Authenticator):
-    def __init__(
+    def __init__(self, name: str) -> None:
-            self,
-            name: str,
-    ) -> None:
         super().__init__(name)
 
     async def username(self) -> str:

PFERD/cli/__init__.py

@@ -1,11 +1,13 @@
 # isort: skip_file
 
 # The order of imports matters because each command module registers itself
-# with the parser from ".parser". Because of this, isort is disabled for this
+# with the parser from ".parser" and the import order affects the order in
+# which they appear in the help. Because of this, isort is disabled for this
 # file. Also, since we're reexporting or just using the side effect of
 # importing itself, we get a few linting warnings, which we're disabling as
 # well.
 
 from . import command_local  # noqa: F401 imported but unused
 from . import command_kit_ilias_web  # noqa: F401 imported but unused
-from .parser import PARSER, load_default_section  # noqa: F401 imported but unused
+from . import command_kit_ipd  # noqa: F401 imported but unused
+from .parser import PARSER, ParserLoadError, load_default_section  # noqa: F401 imported but unused

PFERD/cli/command_kit_ilias_web.py

@@ -4,7 +4,8 @@ from pathlib import Path
 
 from ..crawl.ilias.file_templates import Links
 from ..logging import log
-from .parser import CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, load_crawler, show_value_error
+from .parser import (CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, ParserLoadError, load_crawler,
+                     show_value_error)
 
 SUBPARSER = SUBPARSERS.add_parser(
     "kit-ilias-web",
@@ -38,6 +39,12 @@ GROUP.add_argument(
     action=BooleanOptionalAction,
     help="use the system keyring to store and retrieve passwords"
 )
+GROUP.add_argument(
+    "--credential-file",
+    type=Path,
+    metavar="PATH",
+    help="read username and password from a credential file"
+)
 GROUP.add_argument(
     "--links",
     type=show_value_error(Links.from_string),
@@ -55,6 +62,11 @@ GROUP.add_argument(
     action=BooleanOptionalAction,
     help="crawl and download videos"
 )
+GROUP.add_argument(
+    "--forums",
+    action=BooleanOptionalAction,
+    help="crawl and download forum posts"
+)
 GROUP.add_argument(
     "--http-timeout", "-t",
     type=float,
@@ -83,16 +95,26 @@ def load(
         section["link_redirect_delay"] = str(args.link_redirect_delay)
     if args.videos is not None:
         section["videos"] = "yes" if args.videos else "no"
+    if args.forums is not None:
+        section["forums"] = "yes" if args.forums else "no"
     if args.http_timeout is not None:
         section["http_timeout"] = str(args.http_timeout)
 
     parser["auth:ilias"] = {}
     auth_section = parser["auth:ilias"]
+    if args.credential_file is not None:
+        if args.username is not None:
+            raise ParserLoadError("--credential-file and --username can't be used together")
+        if args.keyring:
+            raise ParserLoadError("--credential-file and --keyring can't be used together")
+        auth_section["type"] = "credential-file"
+        auth_section["path"] = str(args.credential_file)
+    elif args.keyring:
+        auth_section["type"] = "keyring"
+    else:
         auth_section["type"] = "simple"
     if args.username is not None:
         auth_section["username"] = args.username
-    if args.keyring:
-        auth_section["type"] = "keyring"
 
 
 SUBPARSER.set_defaults(command=load)

PFERD/cli/command_kit_ipd.py

@@ -0,0 +1,54 @@
+import argparse
+import configparser
+from pathlib import Path
+
+from ..logging import log
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
+
+SUBPARSER = SUBPARSERS.add_parser(
+    "kit-ipd",
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title="kit ipd crawler arguments",
+    description="arguments for the 'kit-ipd' crawler",
+)
+GROUP.add_argument(
+    "--link-regex",
+    type=str,
+    metavar="REGEX",
+    help="href-matching regex to identify downloadable files"
+)
+GROUP.add_argument(
+    "target",
+    type=str,
+    metavar="TARGET",
+    help="url to crawl"
+)
+GROUP.add_argument(
+    "output",
+    type=Path,
+    metavar="OUTPUT",
+    help="output directory"
+)
+
+
+def load(
+        args: argparse.Namespace,
+        parser: configparser.ConfigParser,
+) -> None:
+    log.explain("Creating config for command 'kit-ipd'")
+
+    parser["crawl:kit-ipd"] = {}
+    section = parser["crawl:kit-ipd"]
+    load_crawler(args, section)
+
+    section["type"] = "kit-ipd"
+    section["target"] = str(args.target)
+    section["output_dir"] = str(args.output)
+    if args.link_regex:
+        section["link_regex"] = str(args.link_regex)
+
+
+SUBPARSER.set_defaults(command=load)

PFERD/cli/parser.py

@@ -8,6 +8,10 @@ from ..output_dir import OnConflict, Redownload
 from ..version import NAME, VERSION
 
 
+class ParserLoadError(Exception):
+    pass
+
+
 # TODO Replace with argparse version when updating to 3.9?
 class BooleanOptionalAction(argparse.Action):
     def __init__(
@@ -177,6 +181,14 @@ PARSER.add_argument(
     help="only execute a single crawler."
     " Can be specified multiple times to execute multiple crawlers"
 )
+PARSER.add_argument(
+    "--skip", "-S",
+    action="append",
+    type=str,
+    metavar="NAME",
+    help="don't execute this particular crawler."
+    " Can be specified multiple times to skip multiple crawlers"
+)
 PARSER.add_argument(
     "--working-dir",
     type=Path,
@@ -203,6 +215,11 @@ PARSER.add_argument(
     action=BooleanOptionalAction,
     help="whether crawlers should share cookies where applicable"
 )
+PARSER.add_argument(
+    "--show-not-deleted",
+    action=BooleanOptionalAction,
+    help="print messages in status and report when PFERD did not delete a local only file"
+)
 
 
 def load_default_section(
@@ -221,6 +238,7 @@ def load_default_section(
         section["report"] = "yes" if args.report else "no"
     if args.share_cookies is not None:
         section["share_cookies"] = "yes" if args.share_cookies else "no"
-
+    if args.show_not_deleted is not None:
+        section["show_not_deleted"] = "yes" if args.show_not_deleted else "no"
 
 SUBPARSERS = PARSER.add_subparsers(title="crawlers")

PFERD/config.py

@@ -69,6 +69,7 @@ class Section:
 
 class DefaultSection(Section):
     def working_dir(self) -> Path:
+        # TODO Change to working dir instead of manually prepending it to paths
         pathstr = self.s.get("working_dir", ".")
         return Path(pathstr).expanduser()
 
@@ -81,6 +82,9 @@ class DefaultSection(Section):
     def report(self) -> bool:
         return self.s.getboolean("report", fallback=True)
 
+    def show_not_deleted(self) -> bool:
+        return self.s.getboolean("show_not_deleted", fallback=True)
+
     def share_cookies(self) -> bool:
         return self.s.getboolean("share_cookies", fallback=True)
 
@@ -119,7 +123,7 @@ class Config:
         # Using config.read_file instead of config.read because config.read
         # would just ignore a missing file and carry on.
         try:
-            with open(path) as f:
+            with open(path, encoding="utf-8") as f:
                 parser.read_file(f, source=str(path))
         except FileNotFoundError:
             raise ConfigLoadError(path, "File does not exist")
@@ -127,6 +131,8 @@ class Config:
             raise ConfigLoadError(path, "That's a directory, not a file")
         except PermissionError:
             raise ConfigLoadError(path, "Insufficient permissions")
+        except UnicodeDecodeError:
+            raise ConfigLoadError(path, "File is not encoded using UTF-8")
 
     def dump(self, path: Optional[Path] = None) -> None:
         """
@@ -153,12 +159,12 @@ class Config:
             try:
                 # x = open for exclusive creation, failing if the file already
                 # exists
-                with open(path, "x") as f:
+                with open(path, "x", encoding="utf-8") as f:
                     self._parser.write(f)
             except FileExistsError:
                 print("That file already exists.")
                 if asyncio.run(prompt_yes_no("Overwrite it?", default=False)):
-                    with open(path, "w") as f:
+                    with open(path, "w", encoding="utf-8") as f:
                         self._parser.write(f)
                 else:
                     raise ConfigDumpError(path, "File already exists")

PFERD/crawl/__init__.py

@@ -3,8 +3,9 @@ from typing import Callable, Dict
 
 from ..auth import Authenticator
 from ..config import Config
-from .crawler import Crawler, CrawlError  # noqa: F401
+from .crawler import Crawler, CrawlError, CrawlerSection  # noqa: F401
 from .ilias import KitIliasWebCrawler, KitIliasWebCrawlerSection
+from .kit_ipd_crawler import KitIpdCrawler, KitIpdCrawlerSection
 from .local_crawler import LocalCrawler, LocalCrawlerSection
 
 CrawlerConstructor = Callable[[
@@ -19,4 +20,6 @@ CRAWLERS: Dict[str, CrawlerConstructor] = {
         LocalCrawler(n, LocalCrawlerSection(s), c),
     "kit-ilias-web": lambda n, s, c, a:
         KitIliasWebCrawler(n, KitIliasWebCrawlerSection(s), c, a),
+    "kit-ipd": lambda n, s, c, a:
+        KitIpdCrawler(n, KitIpdCrawlerSection(s), c),
 }

PFERD/crawl/crawler.py

@@ -1,9 +1,10 @@
 import asyncio
 import os
 from abc import ABC, abstractmethod
+from collections.abc import Awaitable, Coroutine
 from datetime import datetime
 from pathlib import Path, PurePath
-from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set, Tuple, TypeVar
+from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, TypeVar
 
 from ..auth import Authenticator
 from ..config import Config, Section
@@ -47,16 +48,18 @@ def noncritical(f: Wrapped) -> Wrapped:
         try:
             f(*args, **kwargs)
         except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
+            crawler.report.add_warning(str(e))
             log.warn(str(e))
             crawler.error_free = False
-        except:  # noqa: E722 do not use bare 'except'
+        except Exception as e:
             crawler.error_free = False
+            crawler.report.add_error(str(e))
             raise
 
     return wrapper  # type: ignore
 
 
-AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[None]])
+AWrapped = TypeVar("AWrapped", bound=Callable[..., Coroutine[Any, Any, Optional[Any]]])
 
 
 def anoncritical(f: AWrapped) -> AWrapped:
@@ -72,21 +75,25 @@ def anoncritical(f: AWrapped) -> AWrapped:
     Warning: Must only be applied to member functions of the Crawler class!
     """
 
-    async def wrapper(*args: Any, **kwargs: Any) -> None:
+    async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
         if not (args and isinstance(args[0], Crawler)):
             raise RuntimeError("@anoncritical must only applied to Crawler methods")
 
         crawler = args[0]
 
         try:
-            await f(*args, **kwargs)
+            return await f(*args, **kwargs)
         except (CrawlWarning, OutputDirError, MarkDuplicateError, MarkConflictError) as e:
             log.warn(str(e))
             crawler.error_free = False
-        except:  # noqa: E722 do not use bare 'except'
+            crawler.report.add_warning(str(e))
+        except Exception as e:
             crawler.error_free = False
+            crawler.report.add_error(str(e))
             raise
 
+        return None
+
     return wrapper  # type: ignore
 
 
@@ -132,6 +139,15 @@ class DownloadToken(ReusableAsyncContextManager[Tuple[ProgressBar, FileSink]]):
 
 
 class CrawlerSection(Section):
+    def type(self) -> str:
+        value = self.s.get("type")
+        if value is None:
+            self.missing_value("type")
+        return value
+
+    def skip(self) -> bool:
+        return self.s.getboolean("skip", fallback=False)
+
     def output_dir(self, name: str) -> Path:
         # TODO Use removeprefix() after switching to 3.9
         if name.startswith("crawl:"):
@@ -309,6 +325,7 @@ class Crawler(ABC):
             log.explain("Warnings or errors occurred during this run")
             log.explain("Answer: No")
 
+    @anoncritical
     async def run(self) -> None:
         """
         Start the crawling process. Call this function if you want to use a

PFERD/crawl/http_crawler.py

@@ -1,7 +1,8 @@
 import asyncio
+import http.cookies
 import ssl
 from pathlib import Path, PurePath
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import aiohttp
 import certifi
@@ -105,6 +106,25 @@ class HttpCrawler(Crawler):
 
         self._shared_cookie_jar_paths.append(self._cookie_jar_path)
 
+    def _load_cookies_from_file(self, path: Path) -> None:
+        jar: Any = http.cookies.SimpleCookie()
+        with open(path, encoding="utf-8") as f:
+            for i, line in enumerate(f):
+                # Names of headers are case insensitive
+                if line[:11].lower() == "set-cookie:":
+                    jar.load(line[11:])
+                else:
+                    log.explain(f"Line {i} doesn't start with 'Set-Cookie:', ignoring it")
+        self._cookie_jar.update_cookies(jar)
+
+    def _save_cookies_to_file(self, path: Path) -> None:
+        jar: Any = http.cookies.SimpleCookie()
+        for morsel in self._cookie_jar:
+            jar[morsel.key] = morsel
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(jar.output(sep="\n"))
+            f.write("\n")  # A trailing newline is just common courtesy
+
     def _load_cookies(self) -> None:
         log.explain_topic("Loading cookies")
 
@@ -134,7 +154,7 @@ class HttpCrawler(Crawler):
 
         log.explain(f"Loading cookies from {fmt_real_path(cookie_jar_path)}")
         try:
-            self._cookie_jar.load(cookie_jar_path)
+            self._load_cookies_from_file(cookie_jar_path)
         except Exception as e:
             log.explain("Failed to load cookies")
             log.explain(str(e))
@@ -144,7 +164,7 @@ class HttpCrawler(Crawler):
 
         try:
             log.explain(f"Saving cookies to {fmt_real_path(self._cookie_jar_path)}")
-            self._cookie_jar.save(self._cookie_jar_path)
+            self._save_cookies_to_file(self._cookie_jar_path)
         except Exception as e:
             log.warn(f"Failed to save cookies to {fmt_real_path(self._cookie_jar_path)}")
             log.warn(str(e))

PFERD/crawl/ilias/file_templates.py

@@ -1,6 +1,10 @@
 from enum import Enum
 from typing import Optional
 
+import bs4
+
+from PFERD.utils import soupify
+
 _link_template_plain = "{{link}}"
 _link_template_fancy = """
 <!DOCTYPE html>
@@ -94,6 +98,71 @@ _link_template_internet_shortcut = """
 URL={{link}}
 """.strip()
 
+_learning_module_template = """
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>{{name}}</title>
+    </head>
+
+    <style>
+    * {
+        box-sizing: border-box;
+    }
+    .center-flex {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+    }
+    .nav {
+        display: flex;
+        justify-content: space-between;
+    }
+    </style>
+    <body class="center-flex">
+{{body}}
+    </body>
+</html>
+"""
+
+
+def learning_module_template(body: bs4.Tag, name: str, prev: Optional[str], next: Optional[str]) -> str:
+    # Seems to be comments, ignore those.
+    for elem in body.select(".il-copg-mob-fullscreen-modal"):
+        elem.decompose()
+
+    nav_template = """
+        <div class="nav">
+            {{left}}
+            {{right}}
+        </div>
+    """
+    if prev and body.select_one(".ilc_page_lnav_LeftNavigation"):
+        text = body.select_one(".ilc_page_lnav_LeftNavigation").getText().strip()
+        left = f'<a href="{prev}">{text}</a>'
+    else:
+        left = "<span></span>"
+
+    if next and body.select_one(".ilc_page_rnav_RightNavigation"):
+        text = body.select_one(".ilc_page_rnav_RightNavigation").getText().strip()
+        right = f'<a href="{next}">{text}</a>'
+    else:
+        right = "<span></span>"
+
+    if top_nav := body.select_one(".ilc_page_tnav_TopNavigation"):
+        top_nav.replace_with(
+            soupify(nav_template.replace("{{left}}", left).replace("{{right}}", right).encode())
+        )
+
+    if bot_nav := body.select_one(".ilc_page_bnav_BottomNavigation"):
+        bot_nav.replace_with(soupify(nav_template.replace(
+            "{{left}}", left).replace("{{right}}", right).encode())
+        )
+
+    body = body.prettify()
+    return _learning_module_template.replace("{{body}}", body).replace("{{name}}", name)
+
 
 class Links(Enum):
     IGNORE = "ignore"
@@ -102,24 +171,24 @@ class Links(Enum):
     INTERNET_SHORTCUT = "internet-shortcut"
 
     def template(self) -> Optional[str]:
-        if self == self.FANCY:
+        if self == Links.FANCY:
             return _link_template_fancy
-        elif self == self.PLAINTEXT:
+        elif self == Links.PLAINTEXT:
             return _link_template_plain
-        elif self == self.INTERNET_SHORTCUT:
+        elif self == Links.INTERNET_SHORTCUT:
             return _link_template_internet_shortcut
-        elif self == self.IGNORE:
+        elif self == Links.IGNORE:
             return None
         raise ValueError("Missing switch case")
 
     def extension(self) -> Optional[str]:
-        if self == self.FANCY:
+        if self == Links.FANCY:
             return ".html"
-        elif self == self.PLAINTEXT:
+        elif self == Links.PLAINTEXT:
             return ".txt"
-        elif self == self.INTERNET_SHORTCUT:
+        elif self == Links.INTERNET_SHORTCUT:
             return ".url"
-        elif self == self.IGNORE:
+        elif self == Links.IGNORE:
             return None
         raise ValueError("Missing switch case")
 

PFERD/crawl/ilias/ilias_html_cleaner.py

@@ -0,0 +1,91 @@
+from bs4 import BeautifulSoup, Comment, Tag
+
+_STYLE_TAG_CONTENT = """
+    .ilc_text_block_Information {
+      background-color: #f5f7fa;
+    }
+    div.ilc_text_block_Standard {
+      margin-bottom: 10px;
+      margin-top: 10px;
+    }
+    span.ilc_text_inline_Strong {
+      font-weight: bold;
+    }
+
+    .accordion-head {
+      background-color: #f5f7fa;
+      padding: 0.5rem 0;
+    }
+
+    h3 {
+      margin-top: 0.5rem;
+      margin-bottom: 1rem;
+    }
+
+    br.visible-break {
+      margin-bottom: 1rem;
+    }
+
+    article {
+      margin: 0.5rem 0;
+    }
+
+    body {
+      padding: 1em;
+      grid-template-columns: 1fr min(60rem, 90%) 1fr;
+      line-height: 1.2;
+    }
+"""
+
+_ARTICLE_WORTHY_CLASSES = [
+    "ilc_text_block_Information",
+    "ilc_section_Attention",
+    "ilc_section_Link",
+]
+
+
+def insert_base_markup(soup: BeautifulSoup) -> BeautifulSoup:
+    head = soup.new_tag("head")
+    soup.insert(0, head)
+
+    simplecss_link: Tag = soup.new_tag("link")
+    # <link rel="stylesheet" href="https://cdn.simplecss.org/simple.css">
+    simplecss_link["rel"] = "stylesheet"
+    simplecss_link["href"] = "https://cdn.simplecss.org/simple.css"
+    head.append(simplecss_link)
+
+    # Basic style tags for compat
+    style: Tag = soup.new_tag("style")
+    style.append(_STYLE_TAG_CONTENT)
+    head.append(style)
+
+    return soup
+
+
+def clean(soup: BeautifulSoup) -> BeautifulSoup:
+    for block in soup.find_all(class_=lambda x: x in _ARTICLE_WORTHY_CLASSES):
+        block.name = "article"
+
+    for block in soup.find_all("h3"):
+        block.name = "div"
+
+    for block in soup.find_all("h1"):
+        block.name = "h3"
+
+    for block in soup.find_all(class_="ilc_va_ihcap_VAccordIHeadCap"):
+        block.name = "h3"
+        block["class"] += ["accordion-head"]
+
+    for dummy in soup.select(".ilc_text_block_Standard.ilc_Paragraph"):
+        children = list(dummy.children)
+        if not children:
+            dummy.decompose()
+        if len(children) > 1:
+            continue
+        if isinstance(type(children[0]), Comment):
+            dummy.decompose()
+
+    for hrule_imposter in soup.find_all(class_="ilc_section_Separator"):
+        hrule_imposter.insert(0, soup.new_tag("hr"))
+
+    return soup

PFERD/crawl/ilias/kit_ilias_html.py

@@ -3,7 +3,7 @@ import re
 from dataclasses import dataclass
 from datetime import date, datetime, timedelta
 from enum import Enum
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 from urllib.parse import urljoin, urlparse
 
 from bs4 import BeautifulSoup, Tag
@@ -22,7 +22,10 @@ class IliasElementType(Enum):
     FOLDER = "folder"
     FORUM = "forum"
     LINK = "link"
+    LEARNING_MODULE = "learning_module"
+    BOOKING = "booking"
     MEETING = "meeting"
+    SURVEY = "survey"
     VIDEO = "video"
     VIDEO_PLAYER = "video_player"
     VIDEO_FOLDER = "video_folder"
@@ -37,6 +40,45 @@ class IliasPageElement:
     mtime: Optional[datetime] = None
     description: Optional[str] = None
 
+    def id(self) -> str:
+        regexes = [
+            r"eid=(?P<id>[0-9a-z\-]+)",
+            r"file_(?P<id>\d+)",
+            r"ref_id=(?P<id>\d+)",
+            r"target=[a-z]+_(?P<id>\d+)"
+        ]
+
+        for regex in regexes:
+            if match := re.search(regex, self.url):
+                return match.groupdict()["id"]
+
+        # Fall back to URL
+        log.warn(f"Didn't find identity for {self.name} - {self.url}. Please report this.")
+        return self.url
+
+
+@dataclass
+class IliasDownloadForumData:
+    url: str
+    form_data: Dict[str, Union[str, List[str]]]
+    empty: bool
+
+
+@dataclass
+class IliasForumThread:
+    title: str
+    title_tag: Tag
+    content_tag: Tag
+    mtime: Optional[datetime]
+
+
+@dataclass
+class IliasLearningModulePage:
+    title: str
+    content: Tag
+    next_url: Optional[str]
+    previous_url: Optional[str]
+
 
 class IliasPage:
 
@@ -46,6 +88,16 @@ class IliasPage:
         self._page_type = source_element.type if source_element else None
         self._source_name = source_element.name if source_element else ""
 
+    @staticmethod
+    def is_root_page(soup: BeautifulSoup) -> bool:
+        permalink = soup.find(id="current_perma_link")
+        if permalink is None:
+            return False
+        value = permalink.attrs.get("value")
+        if value is None:
+            return False
+        return "goto.php?target=root_" in value
+
     def get_child_elements(self) -> List[IliasPageElement]:
         """
         Return all child page elements you can find here.
@@ -59,14 +111,111 @@ class IliasPage:
         if self._is_exercise_file():
             log.explain("Page is an exercise, searching for elements")
             return self._find_exercise_entries()
+        if self._is_personal_desktop():
+            log.explain("Page is the personal desktop, searching for elements")
+            return self._find_personal_desktop_entries()
+        if self._is_content_page():
+            log.explain("Page is a content page, searching for elements")
+            return self._find_copa_entries()
         log.explain("Page is a normal folder, searching for elements")
         return self._find_normal_entries()
 
-    def get_next_stage_url(self) -> Optional[str]:
+    def get_description(self) -> Optional[BeautifulSoup]:
-        if self._is_ilias_opencast_embedding():
+        def is_interesting_class(name: str) -> bool:
-            return self.get_child_elements()[0].url
+            return name in ["ilCOPageSection", "ilc_Paragraph", "ilc_va_ihcap_VAccordIHeadCap"]
+
+        paragraphs: List[Tag] = self._soup.findAll(class_=is_interesting_class)
+        if not paragraphs:
             return None
 
+        # Extract bits and pieces into a string and parse it again.
+        # This ensures we don't miss anything and weird structures are resolved
+        # somewhat gracefully.
+        raw_html = ""
+        for p in paragraphs:
+            if p.find_parent(class_=is_interesting_class):
+                continue
+
+            # Ignore special listings (like folder groupings)
+            if "ilc_section_Special" in p["class"]:
+                continue
+
+            raw_html += str(p) + "\n"
+        raw_html = f"<body>\n{raw_html}\n</body>"
+
+        return BeautifulSoup(raw_html, "html.parser")
+
+    def get_learning_module_data(self) -> Optional[IliasLearningModulePage]:
+        if not self._is_learning_module_page():
+            return None
+        content = self._soup.select_one("#ilLMPageContent")
+        title = self._soup.select_one(".ilc_page_title_PageTitle").getText().strip()
+        return IliasLearningModulePage(
+            title=title,
+            content=content,
+            next_url=self._find_learning_module_next(),
+            previous_url=self._find_learning_module_prev()
+        )
+
+    def _find_learning_module_next(self) -> Optional[str]:
+        for link in self._soup.select("a.ilc_page_rnavlink_RightNavigationLink"):
+            url = self._abs_url_from_link(link)
+            if "baseClass=ilLMPresentationGUI" not in url:
+                continue
+            return url
+        return None
+
+    def _find_learning_module_prev(self) -> Optional[str]:
+        for link in self._soup.select("a.ilc_page_lnavlink_LeftNavigationLink"):
+            url = self._abs_url_from_link(link)
+            if "baseClass=ilLMPresentationGUI" not in url:
+                continue
+            return url
+        return None
+
+    def get_download_forum_data(self) -> Optional[IliasDownloadForumData]:
+        form = self._soup.find("form", attrs={"action": lambda x: x and "fallbackCmd=showThreads" in x})
+        if not form:
+            return None
+        post_url = self._abs_url_from_relative(form["action"])
+
+        thread_ids = [f["value"] for f in form.find_all(attrs={"name": "thread_ids[]"})]
+
+        form_data: Dict[str, Union[str, List[str]]] = {
+            "thread_ids[]": thread_ids,
+            "selected_cmd2": "html",
+            "select_cmd2": "Ausführen",
+            "selected_cmd": "",
+        }
+
+        return IliasDownloadForumData(url=post_url, form_data=form_data, empty=len(thread_ids) == 0)
+
+    def get_next_stage_element(self) -> Optional[IliasPageElement]:
+        if self._is_forum_page():
+            if "trows=800" in self._page_url:
+                return None
+            log.explain("Requesting *all* forum threads")
+            return self._get_show_max_forum_entries_per_page_url()
+        if self._is_ilias_opencast_embedding():
+            log.explain("Unwrapping opencast embedding")
+            return self.get_child_elements()[0]
+        if self._page_type == IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED:
+            log.explain("Unwrapping video pagination")
+            return self._find_video_entries_paginated()[0]
+        if self._contains_collapsed_future_meetings():
+            log.explain("Requesting *all* future meetings")
+            return self._uncollapse_future_meetings_url()
+        if not self._is_content_tab_selected():
+            return self._select_content_page_url()
+        return None
+
+    def _is_forum_page(self) -> bool:
+        read_more_btn = self._soup.find(
+            "button",
+            attrs={"onclick": lambda x: x and "cmdClass=ilobjforumgui&cmd=markAllRead" in x}
+        )
+        return read_more_btn is not None
+
     def _is_video_player(self) -> bool:
         return "paella_config_file" in str(self._soup)
 
@@ -101,13 +250,59 @@ class IliasPage:
 
         return False
 
+    def _is_personal_desktop(self) -> bool:
+        return self._soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x})
+
+    def _is_content_page(self) -> bool:
+        link = self._soup.find(id="current_perma_link")
+        if not link:
+            return False
+        return "target=copa_" in link.get("value")
+
+    def _is_learning_module_page(self) -> bool:
+        link = self._soup.find(id="current_perma_link")
+        if not link:
+            return False
+        return "target=pg_" in link.get("value")
+
+    def _contains_collapsed_future_meetings(self) -> bool:
+        return self._uncollapse_future_meetings_url() is not None
+
+    def _uncollapse_future_meetings_url(self) -> Optional[IliasPageElement]:
+        element = self._soup.find("a", attrs={"href": lambda x: x and "crs_next_sess=1" in x})
+        if not element:
+            return None
+        link = self._abs_url_from_link(element)
+        return IliasPageElement(IliasElementType.FOLDER, link, "show all meetings")
+
+    def _is_content_tab_selected(self) -> bool:
+        return self._select_content_page_url() is None
+
+    def _select_content_page_url(self) -> Optional[IliasPageElement]:
+        tab = self._soup.find(
+            id="tab_view_content",
+            attrs={"class": lambda x: x is not None and "active" not in x}
+        )
+        # Already selected (or not found)
+        if not tab:
+            return None
+        link = tab.find("a")
+        if link:
+            link = self._abs_url_from_link(link)
+            return IliasPageElement(IliasElementType.FOLDER, link, "select content page")
+
+        _unexpected_html_warning()
+        log.warn_contd(f"Could not find content tab URL on {self._page_url!r}.")
+        log.warn_contd("PFERD might not find content on the course's main page.")
+        return None
+
     def _player_to_video(self) -> List[IliasPageElement]:
         # Fetch the actual video page. This is a small wrapper page initializing a javscript
         # player. Sadly we can not execute that JS. The actual video stream url is nowhere
         # on the page, but defined in a JS object inside a script tag, passed to the player
         # library.
         # We do the impossible and RegEx the stream JSON object out of the page's HTML source
-        regex: re.Pattern[str] = re.compile(
+        regex = re.compile(
             r"({\"streams\"[\s\S]+?),\s*{\"paella_config_file", re.IGNORECASE
         )
         json_match = regex.search(str(self._soup))
@@ -119,10 +314,77 @@ class IliasPage:
 
         # parse it
         json_object = json.loads(json_str)
-        # and fetch the video url!
+        streams = [stream for stream in json_object["streams"]]
-        video_url = json_object["streams"][0]["sources"]["mp4"][0]["src"]
+
+        # and just fetch the lone video url!
+        if len(streams) == 1:
+            video_url = streams[0]["sources"]["mp4"][0]["src"]
             return [IliasPageElement(IliasElementType.VIDEO, video_url, self._source_name)]
 
+        log.explain(f"Found multiple videos for stream at {self._source_name}")
+        items = []
+        for stream in sorted(streams, key=lambda stream: stream["content"]):
+            full_name = f"{self._source_name.replace('.mp4', '')} ({stream['content']}).mp4"
+            video_url = stream["sources"]["mp4"][0]["src"]
+            items.append(IliasPageElement(IliasElementType.VIDEO, video_url, full_name))
+
+        return items
+
+    def _get_show_max_forum_entries_per_page_url(self) -> Optional[IliasPageElement]:
+        correct_link = self._soup.find(
+            "a",
+            attrs={"href": lambda x: x and "trows=800" in x and "cmd=showThreads" in x}
+        )
+
+        if not correct_link:
+            return None
+
+        link = self._abs_url_from_link(correct_link)
+
+        return IliasPageElement(IliasElementType.FORUM, link, "show all forum threads")
+
+    def _find_personal_desktop_entries(self) -> List[IliasPageElement]:
+        items: List[IliasPageElement] = []
+
+        titles: List[Tag] = self._soup.select(".il-item-title")
+        for title in titles:
+            link = title.find("a")
+            name = _sanitize_path_name(link.text.strip())
+            url = self._abs_url_from_link(link)
+
+            type = self._find_type_from_link(name, link, url)
+            if not type:
+                _unexpected_html_warning()
+                log.warn_contd(f"Could not extract type for {link}")
+                continue
+
+            log.explain(f"Found {name!r}")
+
+            if type == IliasElementType.FILE and "_download" not in url:
+                url = re.sub(r"(target=file_\d+)", r"\1_download", url)
+                log.explain("Rewired file URL to include download part")
+
+            items.append(IliasPageElement(type, url, name))
+
+        return items
+
+    def _find_copa_entries(self) -> List[IliasPageElement]:
+        items: List[IliasPageElement] = []
+        links: List[Tag] = self._soup.findAll(class_="ilc_flist_a_FileListItemLink")
+
+        for link in links:
+            url = self._abs_url_from_link(link)
+            name = _sanitize_path_name(link.getText().strip().replace("\t", ""))
+
+            if "file_id" not in url:
+                _unexpected_html_warning()
+                log.warn_contd(f"Found unknown content page item {name!r} with url {url!r}")
+                continue
+
+            items.append(IliasPageElement(IliasElementType.FILE, url, name))
+
+        return items
+
     def _find_video_entries(self) -> List[IliasPageElement]:
         # ILIAS has three stages for video pages
         # 1. The initial dummy page without any videos. This page contains the link to the listing
@@ -180,7 +442,7 @@ class IliasPage:
         """
         # Video start links are marked with an "Abspielen" link
         video_links: List[Tag] = self._soup.findAll(
-            name="a", text=re.compile(r"\s*Abspielen\s*")
+            name="a", text=re.compile(r"\s*(Abspielen|Play)\s*")
         )
 
         results: List[IliasPageElement] = []
@@ -192,11 +454,22 @@ class IliasPage:
 
     def _listed_video_to_element(self, link: Tag) -> IliasPageElement:
         # The link is part of a table with multiple columns, describing metadata.
-        # 6th child (1 indexed) is the modification time string
+        # 6th or 7th child (1 indexed) is the modification time string. Try to find it
+        # by parsing backwards from the end and finding something that looks like a date
+        modification_time = None
+        row: Tag = link.parent.parent.parent
+        column_count = len(row.select("td.std"))
+        for index in range(column_count, 0, -1):
             modification_string = link.parent.parent.parent.select_one(
-            "td.std:nth-child(6)"
+                f"td.std:nth-child({index})"
             ).getText().strip()
+            if re.search(r"\d+\.\d+.\d+ - \d+:\d+", modification_string):
                 modification_time = datetime.strptime(modification_string, "%d.%m.%Y - %H:%M")
+                break
+
+        if modification_time is None:
+            log.warn(f"Could not determine upload time for {link}")
+            modification_time = datetime.now()
 
         title = link.parent.parent.parent.select_one("td.std:nth-child(3)").getText().strip()
         title += ".mp4"
@@ -230,12 +503,16 @@ class IliasPage:
             parent_row: Tag = link.findParent("tr")
             children: List[Tag] = parent_row.findChildren("td")
 
-            # <checkbox> <name> <uploader> <date> <download>
-            #     0         1        2       3        4
             name = _sanitize_path_name(children[1].getText().strip())
-            date = demangle_date(children[3].getText().strip())
-
             log.explain(f"Found exercise detail entry {name!r}")
+
+            for child in reversed(children):
+                date = demangle_date(child.getText().strip(), fail_silently=True)
+                if date is not None:
+                    break
+            if date is None:
+                log.warn(f"Date parsing failed for exercise entry {name!r}")
+
             results.append(IliasPageElement(
                 IliasElementType.FILE,
                 self._abs_url_from_link(link),
@@ -289,7 +566,13 @@ class IliasPage:
 
             # Add each listing as a new
             for listing in file_listings:
-                file_name = _sanitize_path_name(listing.getText().strip())
+                parent_container: Tag = listing.findParent(
+                    "div", attrs={"class": lambda x: x and "form-group" in x}
+                )
+                label_container: Tag = parent_container.find(
+                    attrs={"class": lambda x: x and "control-label" in x}
+                )
+                file_name = _sanitize_path_name(label_container.getText().strip())
                 url = self._abs_url_from_link(listing)
                 log.explain(f"Found exercise detail {file_name!r} at {url}")
                 results.append(IliasPageElement(
@@ -319,6 +602,12 @@ class IliasPage:
             element_type = self._find_type_from_link(element_name, link, abs_url)
             description = self._find_link_description(link)
 
+            # The last meeting on every page is expanded by default.
+            # Its content is then shown inline *and* in the meeting page itself.
+            # We should skip the inline content.
+            if element_type != IliasElementType.MEETING and self._is_in_expanded_meeting(link):
+                continue
+
             if not element_type:
                 continue
             if element_type == IliasElementType.MEETING:
@@ -332,8 +621,30 @@ class IliasPage:
             log.explain(f"Found {element_name!r}")
             result.append(IliasPageElement(element_type, abs_url, element_name, description=description))
 
+        result += self._find_cards()
+
         return result
 
+    def _is_in_expanded_meeting(self, tag: Tag) -> bool:
+        """
+        Returns whether a file is part of an expanded meeting.
+        Has false positives for meetings themselves as their title is also "in the expanded meeting content".
+        It is in the same general div and this whole thing is guesswork.
+        Therefore, you should check for meetings before passing them in this function.
+        """
+        parents: List[Tag] = list(tag.parents)
+        for parent in parents:
+            if not parent.get("class"):
+                continue
+
+            # We should not crawl files under meetings
+            if "ilContainerListItemContentCB" in parent.get("class"):
+                link: Tag = parent.parent.find("a")
+                type = IliasPage._find_type_from_folder_like(link, self._page_url)
+                return type == IliasElementType.MEETING
+
+        return False
+
     def _find_upwards_folder_hierarchy(self, tag: Tag) -> List[str]:
         """
         Interprets accordions and expandable blocks as virtual folders and returns them
@@ -359,7 +670,10 @@ class IliasPage:
                     continue
                 prev: Tag = parent.findPreviousSibling("div")
                 if "ilContainerBlockHeader" in prev.get("class"):
+                    if prev.find("h3"):
                         found_titles.append(prev.find("h3").getText().strip())
+                    else:
+                        found_titles.append(prev.find("h2").getText().strip())
 
             # And this for real accordions
             if "il_VAccordionContentDef" in parent.get("class"):
@@ -414,6 +728,100 @@ class IliasPage:
         log.explain(f"Found file {full_path!r}")
         return IliasPageElement(IliasElementType.FILE, url, full_path, modification_date)
 
+    def _find_cards(self) -> List[IliasPageElement]:
+        result: List[IliasPageElement] = []
+
+        card_titles: List[Tag] = self._soup.select(".card-title a")
+
+        for title in card_titles:
+            url = self._abs_url_from_link(title)
+            name = _sanitize_path_name(title.getText().strip())
+            type = self._find_type_from_card(title)
+
+            if not type:
+                _unexpected_html_warning()
+                log.warn_contd(f"Could not extract type for {title}")
+                continue
+
+            result.append(IliasPageElement(type, url, name))
+
+        card_button_tiles: List[Tag] = self._soup.select(".card-title button")
+
+        for button in card_button_tiles:
+            regex = re.compile(button["id"] + r".*window.open\(['\"](.+?)['\"]")
+            res = regex.search(str(self._soup))
+            if not res:
+                _unexpected_html_warning()
+                log.warn_contd(f"Could not find click handler target for {button}")
+                continue
+            url = self._abs_url_from_relative(res.group(1))
+            name = _sanitize_path_name(button.getText().strip())
+            type = self._find_type_from_card(button)
+            caption_parent = button.findParent(
+                "div",
+                attrs={"class": lambda x: x and "caption" in x},
+            )
+            caption_container = caption_parent.find_next_sibling("div")
+            if caption_container:
+                description = caption_container.getText().strip()
+            else:
+                description = None
+
+            if not type:
+                _unexpected_html_warning()
+                log.warn_contd(f"Could not extract type for {button}")
+                continue
+
+            result.append(IliasPageElement(type, url, name, description=description))
+
+        return result
+
+    def _find_type_from_card(self, card_title: Tag) -> Optional[IliasElementType]:
+        def is_card_root(element: Tag) -> bool:
+            return "il-card" in element["class"] and "thumbnail" in element["class"]
+
+        card_root: Optional[Tag] = None
+
+        # We look for the card root
+        for parent in card_title.parents:
+            if is_card_root(parent):
+                card_root = parent
+                break
+
+        if card_root is None:
+            _unexpected_html_warning()
+            log.warn_contd(f"Tried to figure out element type, but did not find an icon for {card_title}")
+            return None
+
+        icon: Tag = card_root.select_one(".il-card-repository-head .icon")
+
+        if "opencast" in icon["class"] or "xoct" in icon["class"]:
+            return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
+        if "exc" in icon["class"]:
+            return IliasElementType.EXERCISE
+        if "webr" in icon["class"]:
+            return IliasElementType.LINK
+        if "book" in icon["class"]:
+            return IliasElementType.BOOKING
+        if "frm" in icon["class"]:
+            return IliasElementType.FORUM
+        if "sess" in icon["class"]:
+            return IliasElementType.MEETING
+        if "tst" in icon["class"]:
+            return IliasElementType.TEST
+        if "fold" in icon["class"]:
+            return IliasElementType.FOLDER
+        if "copa" in icon["class"]:
+            return IliasElementType.FOLDER
+        if "svy" in icon["class"]:
+            return IliasElementType.SURVEY
+        if "file" in icon["class"]:
+            return IliasElementType.FILE
+
+        _unexpected_html_warning()
+        log.warn_contd(f"Could not extract type from {icon} for card title {card_title}")
+        return None
+
     @staticmethod
     def _find_type_from_link(
             element_name: str,
@@ -429,9 +837,33 @@ class IliasPage:
         if "target=file_" in parsed_url.query:
             return IliasElementType.FILE
 
+        if "target=grp_" in parsed_url.query:
+            return IliasElementType.FOLDER
+
+        if "target=crs_" in parsed_url.query:
+            return IliasElementType.FOLDER
+
+        if "baseClass=ilExerciseHandlerGUI" in parsed_url.query:
+            return IliasElementType.EXERCISE
+
+        if "baseClass=ilLinkResourceHandlerGUI" in parsed_url.query and "calldirectlink" in parsed_url.query:
+            return IliasElementType.LINK
+
+        if "cmd=showThreads" in parsed_url.query or "target=frm_" in parsed_url.query:
+            return IliasElementType.FORUM
+
+        if "cmdClass=ilobjtestgui" in parsed_url.query:
+            return IliasElementType.TEST
+
+        if "baseClass=ilLMPresentationGUI" in parsed_url.query:
+            return IliasElementType.LEARNING_MODULE
+
+        # Booking and Meeting can not be detected based on the link. They do have a ref_id though, so
+        # try to guess it from the image.
+
         # Everything with a ref_id can *probably* be opened to reveal nested things
         # video groups, directories, exercises, etc
-        if "ref_id=" in parsed_url.query:
+        if "ref_id=" in parsed_url.query or "goto.php" in parsed_url.path:
             return IliasPage._find_type_from_folder_like(link_element, url)
 
         _unexpected_html_warning()
@@ -452,7 +884,7 @@ class IliasPage:
         # We look for the outer div of our inner link, to find information around it
         # (mostly the icon)
         for parent in link_element.parents:
-            if "ilContainerListItemOuter" in parent["class"]:
+            if "ilContainerListItemOuter" in parent["class"] or "il-std-item" in parent["class"]:
                 found_parent = parent
                 break
 
@@ -464,13 +896,20 @@ class IliasPage:
         # Find the small descriptive icon to figure out the type
         img_tag: Optional[Tag] = found_parent.select_one("img.ilListItemIcon")
 
+        if img_tag is None:
+            img_tag = found_parent.select_one("img.icon")
+
+        if img_tag is None and found_parent.find("a", attrs={"href": lambda x: x and "crs_next_sess=" in x}):
+            log.explain("Found session expansion button, skipping it as it has no content")
+            return None
+
         if img_tag is None:
             _unexpected_html_warning()
             log.warn_contd(f"Tried to figure out element type, but did not find an image for {url}")
             return None
 
         if "opencast" in str(img_tag["alt"]).lower():
-            return IliasElementType.VIDEO_FOLDER
+            return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
 
         if str(img_tag["src"]).endswith("icon_exc.svg"):
             return IliasElementType.EXERCISE
@@ -478,6 +917,9 @@ class IliasPage:
         if str(img_tag["src"]).endswith("icon_webr.svg"):
             return IliasElementType.LINK
 
+        if str(img_tag["src"]).endswith("icon_book.svg"):
+            return IliasElementType.BOOKING
+
         if str(img_tag["src"]).endswith("frm.svg"):
             return IliasElementType.FORUM
 
@@ -495,23 +937,38 @@ class IliasPage:
         Normalizes meeting names, which have a relative time as their first part,
         to their date in ISO format.
         """
-        date_portion_str = meeting_name.split(" - ")[0]
+
+        # This checks whether we can reach a `:` without passing a `-`
+        if re.search(r"^[^-]+: ", meeting_name):
+            # Meeting name only contains date: "05. Jan 2000:"
+            split_delimiter = ":"
+        else:
+            # Meeting name contains date and start/end times: "05. Jan 2000, 16:00 - 17:30:"
+            split_delimiter = ", "
+
+        # We have a meeting day without time
+        date_portion_str = meeting_name.split(split_delimiter)[0]
         date_portion = demangle_date(date_portion_str)
 
+        # We failed to parse the date, bail out
         if not date_portion:
             return meeting_name
 
-        rest_of_name = meeting_name
+        # Replace the first section with the absolute date
-        if rest_of_name.startswith(date_portion_str):
+        rest_of_name = split_delimiter.join(meeting_name.split(split_delimiter)[1:])
-            rest_of_name = rest_of_name[len(date_portion_str):]
+        return datetime.strftime(date_portion, "%Y-%m-%d") + split_delimiter + rest_of_name
-
-        return datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") + rest_of_name
 
     def _abs_url_from_link(self, link_tag: Tag) -> str:
         """
         Create an absolute url from an <a> tag.
         """
-        return urljoin(self._page_url, link_tag.get("href"))
+        return self._abs_url_from_relative(link_tag.get("href"))
+
+    def _abs_url_from_relative(self, relative_url: str) -> str:
+        """
+        Create an absolute url from a relative URL.
+        """
+        return urljoin(self._page_url, relative_url)
 
 
 def _unexpected_html_warning() -> None:
@@ -522,38 +979,51 @@ german_months = ['Jan', 'Feb', 'Mär', 'Apr', 'Mai', 'Jun', 'Jul', 'Aug', 'Sep',
 english_months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 
 
-def demangle_date(date_str: str) -> Optional[datetime]:
+def demangle_date(date_str: str, fail_silently: bool = False) -> Optional[datetime]:
     """
-    Demangle a given date in one of the following formats:
+    Demangle a given date in one of the following formats (hour/minute part is optional):
     "Gestern, HH:MM"
     "Heute, HH:MM"
     "Morgen, HH:MM"
     "dd. mon yyyy, HH:MM
     """
     try:
+        # Normalize whitespace because users
         date_str = re.sub(r"\s+", " ", date_str)
+
         date_str = re.sub("Gestern|Yesterday", _format_date_english(_yesterday()), date_str, re.I)
         date_str = re.sub("Heute|Today", _format_date_english(date.today()), date_str, re.I)
         date_str = re.sub("Morgen|Tomorrow",  _format_date_english(_tomorrow()), date_str, re.I)
+        date_str = date_str.strip()
         for german, english in zip(german_months, english_months):
             date_str = date_str.replace(german, english)
             # Remove trailing dots for abbreviations, e.g. "20. Apr. 2020" -> "20. Apr 2020"
             date_str = date_str.replace(english + ".", english)
 
-        # We now have a nice english String in the format: "dd. mmm yyyy, hh:mm"
+        # We now have a nice english String in the format: "dd. mmm yyyy, hh:mm" or "dd. mmm yyyy"
+
+        # Check if we have a time as well
+        if ", " in date_str:
             day_part, time_part = date_str.split(",")
+        else:
+            day_part = date_str.split(",")[0]
+            time_part = None
+
         day_str, month_str, year_str = day_part.split(" ")
 
         day = int(day_str.strip().replace(".", ""))
         month = english_months.index(month_str.strip()) + 1
         year = int(year_str.strip())
 
+        if time_part:
             hour_str, minute_str = time_part.split(":")
             hour = int(hour_str)
             minute = int(minute_str)
-
             return datetime(year, month, day, hour, minute)
+
+        return datetime(year, month, day)
     except Exception:
+        if not fail_silently:
             log.warn(f"Date parsing failed for {date_str!r}")
         return None
 
@@ -573,3 +1043,45 @@ def _tomorrow() -> date:
 
 def _sanitize_path_name(name: str) -> str:
     return name.replace("/", "-").replace("\\", "-").strip()
+
+
+def parse_ilias_forum_export(forum_export: BeautifulSoup) -> List[IliasForumThread]:
+    elements = []
+    for p in forum_export.select("body > p"):
+        title_tag = p
+        content_tag = p.find_next_sibling("ul")
+
+        if not content_tag:
+            # ILIAS allows users to delete the initial post while keeping the thread open
+            # This produces empty threads without *any* content.
+            # I am not sure why you would want this, but ILIAS makes it easy to do.
+            continue
+
+        title = p.find("b").text
+        if ":" in title:
+            title = title[title.find(":") + 1:]
+        title = title.strip()
+        mtime = _guess_timestamp_from_forum_post_content(content_tag)
+        elements.append(IliasForumThread(title, title_tag, content_tag, mtime))
+
+    return elements
+
+
+def _guess_timestamp_from_forum_post_content(content: Tag) -> Optional[datetime]:
+    posts: Optional[Tag] = content.select(".ilFrmPostHeader > span.small")
+    if not posts:
+        return None
+
+    newest_date: Optional[datetime] = None
+
+    for post in posts:
+        text = post.text.strip()
+        text = text[text.rfind("|") + 1:]
+        date = demangle_date(text, fail_silently=True)
+        if not date:
+            continue
+
+        if not newest_date or newest_date < date:
+            newest_date = date
+
+    return newest_date

PFERD/crawl/ilias/kit_ilias_web_crawler.py

@@ -1,9 +1,14 @@
 import asyncio
+import base64
+import os
 import re
+from collections.abc import Awaitable, Coroutine
 from pathlib import PurePath
-from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, TypeVar, Union
+from typing import Any, Callable, Dict, List, Literal, Optional, Set, Union, cast
+from urllib.parse import urljoin
 
 import aiohttp
+import yarl
 from aiohttp import hdrs
 from bs4 import BeautifulSoup, Tag
 
@@ -12,16 +17,23 @@ from ...config import Config
 from ...logging import ProgressBar, log
 from ...output_dir import FileSink, Redownload
 from ...utils import fmt_path, soupify, url_set_query_param
-from ..crawler import CrawlError, CrawlWarning, anoncritical
+from ..crawler import AWrapped, CrawlError, CrawlToken, CrawlWarning, DownloadToken, anoncritical
 from ..http_crawler import HttpCrawler, HttpCrawlerSection
-from .file_templates import Links
+from .file_templates import Links, learning_module_template
-from .kit_ilias_html import IliasElementType, IliasPage, IliasPageElement
+from .ilias_html_cleaner import clean, insert_base_markup
+from .kit_ilias_html import (IliasElementType, IliasForumThread, IliasLearningModulePage, IliasPage,
+                             IliasPageElement, _sanitize_path_name, parse_ilias_forum_export)
 
 TargetType = Union[str, int]
 
+_ILIAS_URL = "https://ilias.studium.kit.edu"
+
+
+class KitShibbolethBackgroundLoginSuccessful():
+    pass
+
 
 class KitIliasWebCrawlerSection(HttpCrawlerSection):
-
     def target(self) -> TargetType:
         target = self.s.get("target")
         if not target:
@@ -33,7 +45,7 @@ class KitIliasWebCrawlerSection(HttpCrawlerSection):
         if target == "desktop":
             # Full personal desktop
             return target
-        if target.startswith("https://ilias.studium.kit.edu"):
+        if target.startswith(_ILIAS_URL):
             # ILIAS URL
             return target
 
@@ -65,6 +77,9 @@ class KitIliasWebCrawlerSection(HttpCrawlerSection):
     def videos(self) -> bool:
         return self.s.getboolean("videos", fallback=False)
 
+    def forums(self) -> bool:
+        return self.s.getboolean("forums", fallback=False)
+
 
 _DIRECTORY_PAGES: Set[IliasElementType] = set([
     IliasElementType.EXERCISE,
@@ -82,17 +97,14 @@ _VIDEO_ELEMENTS: Set[IliasElementType] = set([
     IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED,
 ])
 
-AWrapped = TypeVar("AWrapped", bound=Callable[..., Awaitable[None]])
 
-
+def _iorepeat(attempts: int, name: str, failure_is_error: bool = False) -> Callable[[AWrapped], AWrapped]:
-def _iorepeat(attempts: int, name: str) -> Callable[[AWrapped], AWrapped]:
     def decorator(f: AWrapped) -> AWrapped:
-        async def wrapper(*args: Any, **kwargs: Any) -> None:
+        async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
             last_exception: Optional[BaseException] = None
             for round in range(attempts):
                 try:
-                    await f(*args, **kwargs)
+                    return await f(*args, **kwargs)
-                    return
                 except aiohttp.ContentTypeError:  # invalid content type
                     raise CrawlWarning("ILIAS returned an invalid content type")
                 except aiohttp.TooManyRedirects:
@@ -107,6 +119,9 @@ def _iorepeat(attempts: int, name: str) -> Callable[[AWrapped], AWrapped]:
 
             if last_exception:
                 message = f"Error in I/O Operation: {last_exception}"
+                if failure_is_error:
+                    raise CrawlError(message) from last_exception
+                else:
                     raise CrawlWarning(message) from last_exception
             raise CrawlError("Impossible return in ilias _iorepeat")
 
@@ -164,17 +179,25 @@ class KitIliasWebCrawler(HttpCrawler):
         auth = section.auth(authenticators)
         super().__init__(name, section, config, shared_auth=auth)
 
+        if section.tasks() > 1:
+            log.warn("""
+Please avoid using too many parallel requests as these are the KIT ILIAS
+instance's greatest bottleneck.
+            """.strip())
+
         self._shibboleth_login = KitShibbolethLogin(
             auth,
             section.tfa_auth(authenticators),
         )
 
-        self._base_url = "https://ilias.studium.kit.edu"
+        self._base_url = _ILIAS_URL
 
         self._target = section.target()
         self._link_file_redirect_delay = section.link_redirect_delay()
         self._links = section.links()
         self._videos = section.videos()
+        self._forums = section.forums()
+        self._visited_urls: Dict[str, PurePath] = dict()
 
     async def _run(self) -> None:
         if isinstance(self._target, int):
@@ -196,7 +219,9 @@ class KitIliasWebCrawler(HttpCrawler):
         await self._crawl_url(root_url, expected_id=course_id)
 
     async def _crawl_desktop(self) -> None:
-        await self._crawl_url(self._base_url)
+        appendix = r"ILIAS\PersonalDesktop\PDMainBarProvider|mm_pd_sel_items"
+        appendix = appendix.encode("ASCII").hex()
+        await self._crawl_url(self._base_url + "/gs_content.php?item=" + appendix)
 
     async def _crawl_url(self, url: str, expected_id: Optional[int] = None) -> None:
         maybe_cl = await self.crawl(PurePath("."))
@@ -205,102 +230,195 @@ class KitIliasWebCrawler(HttpCrawler):
         cl = maybe_cl  # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
 
         elements: List[IliasPageElement] = []
+        # A list as variable redefinitions are not propagated to outer scopes
+        description: List[BeautifulSoup] = []
 
         @_iorepeat(3, "crawling url")
         async def gather_elements() -> None:
             elements.clear()
             async with cl:
-                soup = await self._get_page(url)
+                next_stage_url: Optional[str] = url
+                current_parent = None
 
-                if expected_id is not None:
+                # Duplicated code, but the root page is special - we want to avoid fetching it twice!
+                while next_stage_url:
+                    soup = await self._get_page(next_stage_url, root_page_allowed=True)
+
+                    if current_parent is None and expected_id is not None:
                         perma_link_element: Tag = soup.find(id="current_perma_link")
                         if not perma_link_element or "crs_" not in perma_link_element.get("value"):
                             raise CrawlError("Invalid course id? Didn't find anything looking like a course")
 
-                # Duplicated code, but the root page is special - we want to avoid fetching it twice!
+                    log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
-                log.explain_topic("Parsing root HTML page")
+                    log.explain(f"URL: {next_stage_url}")
-                log.explain(f"URL: {url}")
+                    page = IliasPage(soup, next_stage_url, current_parent)
-                page = IliasPage(soup, url, None)
+                    if next_element := page.get_next_stage_element():
+                        current_parent = next_element
+                        next_stage_url = next_element.url
+                    else:
+                        next_stage_url = None
+
                 elements.extend(page.get_child_elements())
+                if description_string := page.get_description():
+                    description.append(description_string)
 
         # Fill up our task list with the found elements
         await gather_elements()
-        tasks = [self._handle_ilias_element(PurePath("."), element) for element in elements]
+
+        if description:
+            await self._download_description(PurePath("."), description[0])
+
+        elements.sort(key=lambda e: e.id())
+
+        tasks: List[Awaitable[None]] = []
+        for element in elements:
+            if handle := await self._handle_ilias_element(PurePath("."), element):
+                tasks.append(asyncio.create_task(handle))
 
         # And execute them
         await self.gather(tasks)
 
-    async def _handle_ilias_page(self, url: str, parent: IliasPageElement, path: PurePath) -> None:
+    async def _handle_ilias_page(
+        self,
+        url: str,
+        parent: IliasPageElement,
+        path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
         maybe_cl = await self.crawl(path)
         if not maybe_cl:
-            return
+            return None
-        cl = maybe_cl  # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
+        return self._crawl_ilias_page(url, parent, maybe_cl)
 
+    @anoncritical
+    async def _crawl_ilias_page(
+        self,
+        url: str,
+        parent: IliasPageElement,
+        cl: CrawlToken,
+    ) -> None:
         elements: List[IliasPageElement] = []
+        # A list as variable redefinitions are not propagated to outer scopes
+        description: List[BeautifulSoup] = []
 
         @_iorepeat(3, "crawling folder")
         async def gather_elements() -> None:
             elements.clear()
             async with cl:
                 next_stage_url: Optional[str] = url
+                current_parent = parent
 
                 while next_stage_url:
                     soup = await self._get_page(next_stage_url)
-                    log.explain_topic(f"Parsing HTML page for {fmt_path(path)}")
+                    log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
                     log.explain(f"URL: {next_stage_url}")
-                    page = IliasPage(soup, url, parent)
+                    page = IliasPage(soup, next_stage_url, current_parent)
-                    next_stage_url = page.get_next_stage_url()
+                    if next_element := page.get_next_stage_element():
+                        current_parent = next_element
+                        next_stage_url = next_element.url
+                    else:
+                        next_stage_url = None
 
                 elements.extend(page.get_child_elements())
+                if description_string := page.get_description():
+                    description.append(description_string)
 
         # Fill up our task list with the found elements
         await gather_elements()
-        tasks = [self._handle_ilias_element(cl.path, element) for element in elements]
+
+        if description:
+            await self._download_description(cl.path, description[0])
+
+        elements.sort(key=lambda e: e.id())
+
+        tasks: List[Awaitable[None]] = []
+        for element in elements:
+            if handle := await self._handle_ilias_element(cl.path, element):
+                tasks.append(asyncio.create_task(handle))
 
         # And execute them
         await self.gather(tasks)
 
+    # These decorators only apply *to this method* and *NOT* to the returned
+    # awaitables!
+    # This method does not await the handlers but returns them instead.
+    # This ensures one level is handled at a time and name deduplication
+    # works correctly.
     @anoncritical
-    # Shouldn't happen but we also really don't want to let I/O errors bubble up to anoncritical.
+    async def _handle_ilias_element(
-    # If that happens we will be terminated as anoncritical doesn't tream them as non-critical.
+        self,
-    @_wrap_io_in_warning("handling ilias element")
+        parent_path: PurePath,
-    async def _handle_ilias_element(self, parent_path: PurePath, element: IliasPageElement) -> None:
+        element: IliasPageElement,
+    ) -> Optional[Coroutine[Any, Any, None]]:
+        if element.url in self._visited_urls:
+            raise CrawlWarning(
+                f"Found second path to element {element.name!r} at {element.url!r}. "
+                + f"First path: {fmt_path(self._visited_urls[element.url])}. "
+                + f"Second path: {fmt_path(parent_path)}."
+            )
+        self._visited_urls[element.url] = parent_path
+
         element_path = PurePath(parent_path, element.name)
 
         if element.type in _VIDEO_ELEMENTS:
-            log.explain_topic(f"Decision: Crawl video element {fmt_path(element_path)}")
             if not self._videos:
-                log.explain("Video crawling is disabled")
+                log.status(
-                log.explain("Answer: no")
+                    "[bold bright_black]",
-                return
+                    "Ignored",
-            else:
+                    fmt_path(element_path),
-                log.explain("Video crawling is enabled")
+                    "[bright_black](enable with option 'videos')"
-                log.explain("Answer: yes")
+                )
+                return None
 
         if element.type == IliasElementType.FILE:
-            await self._download_file(element, element_path)
+            return await self._handle_file(element, element_path)
         elif element.type == IliasElementType.FORUM:
-            log.explain_topic(f"Decision: Crawl {fmt_path(element_path)}")
+            if not self._forums:
-            log.explain("Forums are not supported")
+                log.status(
-            log.explain("Answer: No")
+                    "[bold bright_black]",
+                    "Ignored",
+                    fmt_path(element_path),
+                    "[bright_black](enable with option 'forums')"
+                )
+                return None
+            return await self._handle_forum(element, element_path)
         elif element.type == IliasElementType.TEST:
-            log.explain_topic(f"Decision: Crawl {fmt_path(element_path)}")
+            log.status(
-            log.explain("Tests contain no relevant files")
+                "[bold bright_black]",
-            log.explain("Answer: No")
+                "Ignored",
+                fmt_path(element_path),
+                "[bright_black](tests contain no relevant data)"
+            )
+            return None
+        elif element.type == IliasElementType.SURVEY:
+            log.status(
+                "[bold bright_black]",
+                "Ignored",
+                fmt_path(element_path),
+                "[bright_black](surveys contain no relevant data)"
+            )
+            return None
+        elif element.type == IliasElementType.LEARNING_MODULE:
+            return await self._handle_learning_module(element, element_path)
         elif element.type == IliasElementType.LINK:
-            await self._download_link(element, element_path)
+            return await self._handle_link(element, element_path)
+        elif element.type == IliasElementType.BOOKING:
+            return await self._handle_booking(element, element_path)
         elif element.type == IliasElementType.VIDEO:
-            await self._download_file(element, element_path)
+            return await self._handle_file(element, element_path)
         elif element.type == IliasElementType.VIDEO_PLAYER:
-            await self._download_video(element, element_path)
+            return await self._handle_video(element, element_path)
         elif element.type in _DIRECTORY_PAGES:
-            await self._handle_ilias_page(element.url, element, element_path)
+            return await self._handle_ilias_page(element.url, element, element_path)
         else:
             # This will retry it a few times, failing everytime. It doesn't make any network
             # requests, so that's fine.
             raise CrawlWarning(f"Unknown element type: {element.type!r}")
 
-    async def _download_link(self, element: IliasPageElement, element_path: PurePath) -> None:
+    async def _handle_link(
+        self,
+        element: IliasPageElement,
+        element_path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
         log.explain_topic(f"Decision: Crawl Link {fmt_path(element_path)}")
         log.explain(f"Links type is {self._links}")
 
@@ -308,32 +426,87 @@ class KitIliasWebCrawler(HttpCrawler):
         link_extension = self._links.extension()
         if not link_template_maybe or not link_extension:
             log.explain("Answer: No")
-            return
+            return None
         else:
             log.explain("Answer: Yes")
-        link_template = link_template_maybe
         element_path = element_path.with_name(element_path.name + link_extension)
 
         maybe_dl = await self.download(element_path, mtime=element.mtime)
         if not maybe_dl:
-            return
+            return None
-        dl = maybe_dl  # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
 
+        return self._download_link(element, link_template_maybe, maybe_dl)
+
+    @anoncritical
     @_iorepeat(3, "resolving link")
-        async def impl() -> None:
+    async def _download_link(self, element: IliasPageElement, link_template: str, dl: DownloadToken) -> None:
         async with dl as (bar, sink):
             export_url = element.url.replace("cmd=calldirectlink", "cmd=exportHTML")
             real_url = await self._resolve_link_target(export_url)
+            self._write_link_content(link_template, real_url, element.name, element.description, sink)
 
+    def _write_link_content(
+        self,
+        link_template: str,
+        url: str,
+        name: str,
+        description: Optional[str],
+        sink: FileSink,
+    ) -> None:
         content = link_template
-                content = content.replace("{{link}}", real_url)
+        content = content.replace("{{link}}", url)
-                content = content.replace("{{name}}", element.name)
+        content = content.replace("{{name}}", name)
-                content = content.replace("{{description}}", str(element.description))
+        content = content.replace("{{description}}", str(description))
         content = content.replace("{{redirect_delay}}", str(self._link_file_redirect_delay))
         sink.file.write(content.encode("utf-8"))
         sink.done()
 
-        await impl()
+    async def _handle_booking(
+        self,
+        element: IliasPageElement,
+        element_path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
+        log.explain_topic(f"Decision: Crawl Booking Link {fmt_path(element_path)}")
+        log.explain(f"Links type is {self._links}")
+
+        link_template_maybe = self._links.template()
+        link_extension = self._links.extension()
+        if not link_template_maybe or not link_extension:
+            log.explain("Answer: No")
+            return None
+        else:
+            log.explain("Answer: Yes")
+        element_path = element_path.with_name(element_path.name + link_extension)
+
+        maybe_dl = await self.download(element_path, mtime=element.mtime)
+        if not maybe_dl:
+            return None
+
+        return self._download_booking(element, link_template_maybe, maybe_dl)
+
+    @anoncritical
+    @_iorepeat(1, "downloading description")
+    async def _download_description(self, parent_path: PurePath, description: BeautifulSoup) -> None:
+        path = parent_path / "Description.html"
+        dl = await self.download(path, redownload=Redownload.ALWAYS)
+        if not dl:
+            return
+
+        async with dl as (bar, sink):
+            description = clean(insert_base_markup(description))
+            sink.file.write(description.prettify().encode("utf-8"))
+            sink.done()
+
+    @anoncritical
+    @_iorepeat(3, "resolving booking")
+    async def _download_booking(
+        self,
+        element: IliasPageElement,
+        link_template: str,
+        dl: DownloadToken,
+    ) -> None:
+        async with dl as (bar, sink):
+            self._write_link_content(link_template, element.url, element.name, element.description, sink)
 
     async def _resolve_link_target(self, export_url: str) -> str:
         async with self.session.get(export_url, allow_redirects=False) as resp:
@@ -341,7 +514,7 @@ class KitIliasWebCrawler(HttpCrawler):
             if hdrs.LOCATION not in resp.headers:
                 return soupify(await resp.read()).select_one("a").get("href").strip()
 
-        self._authenticate()
+        await self._authenticate()
 
         async with self.session.get(export_url, allow_redirects=False) as resp:
             # No redirect means we were authenticated
@@ -350,40 +523,131 @@ class KitIliasWebCrawler(HttpCrawler):
 
         raise CrawlError("resolve_link_target failed even after authenticating")
 
-    async def _download_video(self, element: IliasPageElement, element_path: PurePath) -> None:
+    async def _handle_video(
-        # Videos will NOT be redownloaded - their content doesn't really change and they are chunky
+        self,
-        maybe_dl = await self.download(element_path, mtime=element.mtime, redownload=Redownload.NEVER)
+        element: IliasPageElement,
-        if not maybe_dl:
+        element_path: PurePath,
-            return
+    ) -> Optional[Coroutine[Any, Any, None]]:
-        dl = maybe_dl  # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
+        # Copy old mapping as it is likely still relevant
+        if self.prev_report:
+            self.report.add_custom_value(
+                str(element_path),
+                self.prev_report.get_custom_value(str(element_path))
+            )
 
+        # A video might contain other videos, so let's "crawl" the video first
+        # to ensure rate limits apply. This must be a download as *this token*
+        # is re-used if the video consists of a single stream. In that case the
+        # file name is used and *not* the stream name the ilias html parser reported
+        # to ensure backwards compatibility.
+        maybe_dl = await self.download(element_path, mtime=element.mtime, redownload=Redownload.ALWAYS)
+
+        # If we do not want to crawl it (user filter) or we have every file
+        # from the cached mapping already, we can ignore this and bail
+        if not maybe_dl or self._all_videos_locally_present(element_path):
+            # Mark all existing cideos as known so they do not get deleted
+            # during dleanup. We "downloaded" them, just without actually making
+            # a network request as we assumed they did not change.
+            for video in self._previous_contained_videos(element_path):
+                await self.download(video)
+
+            return None
+
+        return self._download_video(element_path, element, maybe_dl)
+
+    def _previous_contained_videos(self, video_path: PurePath) -> List[PurePath]:
+        if not self.prev_report:
+            return []
+        custom_value = self.prev_report.get_custom_value(str(video_path))
+        if not custom_value:
+            return []
+        names = cast(List[str], custom_value)
+        folder = video_path.parent
+        return [PurePath(folder, name) for name in names]
+
+    def _all_videos_locally_present(self, video_path: PurePath) -> bool:
+        if contained_videos := self._previous_contained_videos(video_path):
+            log.explain_topic(f"Checking local cache for video {video_path.name}")
+            all_found_locally = True
+            for video in contained_videos:
+                transformed_path = self._to_local_video_path(video)
+                if transformed_path:
+                    exists_locally = self._output_dir.resolve(transformed_path).exists()
+                    all_found_locally = all_found_locally and exists_locally
+            if all_found_locally:
+                log.explain("Found all videos locally, skipping enumeration request")
+                return True
+            log.explain("Missing at least one video, continuing with requests!")
+        return False
+
+    def _to_local_video_path(self, path: PurePath) -> Optional[PurePath]:
+        if transformed := self._transformer.transform(path):
+            return self._deduplicator.fixup_path(transformed)
+        return None
+
+    @anoncritical
     @_iorepeat(3, "downloading video")
-        async def impl() -> None:
+    async def _download_video(
-            assert dl  # The function is only reached when dl is not None
+        self,
+        original_path: PurePath,
+        element: IliasPageElement,
+        dl: DownloadToken
+    ) -> None:
+        stream_elements: List[IliasPageElement] = []
         async with dl as (bar, sink):
             page = IliasPage(await self._get_page(element.url), element.url, element)
-                real_element = page.get_child_elements()[0]
+            stream_elements = page.get_child_elements()
 
-                log.explain(f"Streaming video from real url {real_element.url}")
+            if len(stream_elements) > 1:
+                log.explain(f"Found multiple video streams for {element.name}")
+            else:
+                log.explain(f"Using single video mode for {element.name}")
+                stream_element = stream_elements[0]
 
-                await self._stream_from_url(real_element.url, sink, bar, is_video=True)
+                transformed_path = self._to_local_video_path(original_path)
+                if not transformed_path:
+                    raise CrawlError(f"Download returned a path but transform did not for {original_path}")
 
-        await impl()
+                # We do not have a local cache yet
+                if self._output_dir.resolve(transformed_path).exists():
+                    log.explain(f"Video for {element.name} existed locally")
+                else:
+                    await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
+                self.report.add_custom_value(str(original_path), [original_path.name])
+                return
 
-    async def _download_file(self, element: IliasPageElement, element_path: PurePath) -> None:
+        contained_video_paths: List[str] = []
+
+        for stream_element in stream_elements:
+            video_path = original_path.parent / stream_element.name
+            contained_video_paths.append(str(video_path))
+
+            maybe_dl = await self.download(video_path, mtime=element.mtime, redownload=Redownload.NEVER)
+            if not maybe_dl:
+                continue
+            async with maybe_dl as (bar, sink):
+                log.explain(f"Streaming video from real url {stream_element.url}")
+                await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
+
+        self.report.add_custom_value(str(original_path), contained_video_paths)
+
+    async def _handle_file(
+        self,
+        element: IliasPageElement,
+        element_path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
         maybe_dl = await self.download(element_path, mtime=element.mtime)
         if not maybe_dl:
-            return
+            return None
-        dl = maybe_dl  # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
+        return self._download_file(element, maybe_dl)
 
+    @anoncritical
     @_iorepeat(3, "downloading file")
-        async def impl() -> None:
+    async def _download_file(self, element: IliasPageElement, dl: DownloadToken) -> None:
         assert dl  # The function is only reached when dl is not None
         async with dl as (bar, sink):
             await self._stream_from_url(element.url, sink, bar, is_video=False)
 
-        await impl()
-
     async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar, is_video: bool) -> None:
         async def try_stream() -> bool:
             async with self.session.get(url, allow_redirects=is_video) as resp:
@@ -414,12 +678,207 @@ class KitIliasWebCrawler(HttpCrawler):
         if not await try_stream():
             raise CrawlError("File streaming failed after authenticate()")
 
-    async def _get_page(self, url: str) -> BeautifulSoup:
+    async def _handle_forum(
+        self,
+        element: IliasPageElement,
+        element_path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
+        maybe_cl = await self.crawl(element_path)
+        if not maybe_cl:
+            return None
+        return self._crawl_forum(element, maybe_cl)
+
+    @_iorepeat(3, "crawling forum")
+    @anoncritical
+    async def _crawl_forum(self, element: IliasPageElement, cl: CrawlToken) -> None:
+        elements: List[IliasForumThread] = []
+
+        async with cl:
+            next_stage_url = element.url
+            while next_stage_url:
+                log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
+                log.explain(f"URL: {next_stage_url}")
+
+                soup = await self._get_page(next_stage_url)
+                page = IliasPage(soup, next_stage_url, None)
+
+                if next := page.get_next_stage_element():
+                    next_stage_url = next.url
+                else:
+                    break
+
+            download_data = page.get_download_forum_data()
+            if not download_data:
+                raise CrawlWarning("Failed to extract forum data")
+            if download_data.empty:
+                log.explain("Forum had no threads")
+                elements = []
+                return
+            html = await self._post_authenticated(download_data.url, download_data.form_data)
+            elements = parse_ilias_forum_export(soupify(html))
+
+        elements.sort(key=lambda elem: elem.title)
+
+        tasks: List[Awaitable[None]] = []
+        for elem in elements:
+            tasks.append(asyncio.create_task(self._download_forum_thread(cl.path, elem)))
+
+        # And execute them
+        await self.gather(tasks)
+
+    @anoncritical
+    @_iorepeat(3, "saving forum thread")
+    async def _download_forum_thread(
+        self,
+        parent_path: PurePath,
+        element: IliasForumThread,
+    ) -> None:
+        path = parent_path / (_sanitize_path_name(element.title) + ".html")
+        maybe_dl = await self.download(path, mtime=element.mtime)
+        if not maybe_dl:
+            return
+
+        async with maybe_dl as (bar, sink):
+            content = element.title_tag.prettify()
+            content += element.content_tag.prettify()
+            sink.file.write(content.encode("utf-8"))
+            sink.done()
+
+    async def _handle_learning_module(
+        self,
+        element: IliasPageElement,
+        element_path: PurePath,
+    ) -> Optional[Coroutine[Any, Any, None]]:
+        maybe_cl = await self.crawl(element_path)
+        if not maybe_cl:
+            return None
+        return self._crawl_learning_module(element, maybe_cl)
+
+    @_iorepeat(3, "crawling learning module")
+    @anoncritical
+    async def _crawl_learning_module(self, element: IliasPageElement, cl: CrawlToken) -> None:
+        elements: List[IliasLearningModulePage] = []
+
+        async with cl:
+            log.explain_topic(f"Parsing initial HTML page for {fmt_path(cl.path)}")
+            log.explain(f"URL: {element.url}")
+            soup = await self._get_page(element.url)
+            page = IliasPage(soup, element.url, None)
+            if next := page.get_learning_module_data():
+                elements.extend(await self._crawl_learning_module_direction(
+                    cl.path, next.previous_url, "left"
+                ))
+                elements.append(next)
+                elements.extend(await self._crawl_learning_module_direction(
+                    cl.path, next.next_url, "right"
+                ))
+
+        # Reflect their natural ordering in the file names
+        for index, lm_element in enumerate(elements):
+            lm_element.title = f"{index:02}_{lm_element.title}"
+
+        tasks: List[Awaitable[None]] = []
+        for index, elem in enumerate(elements):
+            prev_url = elements[index - 1].title if index > 0 else None
+            next_url = elements[index + 1].title if index < len(elements) - 1 else None
+            tasks.append(asyncio.create_task(
+                self._download_learning_module_page(cl.path, elem, prev_url, next_url)
+            ))
+
+        # And execute them
+        await self.gather(tasks)
+
+    async def _crawl_learning_module_direction(
+        self,
+        path: PurePath,
+        start_url: Optional[str],
+        dir: Union[Literal["left"], Literal["right"]]
+    ) -> List[IliasLearningModulePage]:
+        elements: List[IliasLearningModulePage] = []
+
+        if not start_url:
+            return elements
+
+        next_element_url: Optional[str] = start_url
+        counter = 0
+        while next_element_url:
+            log.explain_topic(f"Parsing HTML page for {fmt_path(path)} ({dir}-{counter})")
+            log.explain(f"URL: {next_element_url}")
+            soup = await self._get_page(next_element_url)
+            page = IliasPage(soup, next_element_url, None)
+            if next := page.get_learning_module_data():
+                elements.append(next)
+                if dir == "left":
+                    next_element_url = next.previous_url
+                else:
+                    next_element_url = next.next_url
+            counter += 1
+
+        return elements
+
+    @anoncritical
+    @_iorepeat(3, "saving learning module page")
+    async def _download_learning_module_page(
+        self,
+        parent_path: PurePath,
+        element: IliasLearningModulePage,
+        prev: Optional[str],
+        next: Optional[str]
+    ) -> None:
+        path = parent_path / (_sanitize_path_name(element.title) + ".html")
+        maybe_dl = await self.download(path)
+        if not maybe_dl:
+            return
+        my_path = self._transformer.transform(maybe_dl.path)
+        if not my_path:
+            return
+
+        if prev:
+            prev_p = self._transformer.transform(parent_path / (_sanitize_path_name(prev) + ".html"))
+            if prev_p:
+                prev = os.path.relpath(prev_p, my_path.parent)
+            else:
+                prev = None
+        if next:
+            next_p = self._transformer.transform(parent_path / (_sanitize_path_name(next) + ".html"))
+            if next_p:
+                next = os.path.relpath(next_p, my_path.parent)
+            else:
+                next = None
+
+        async with maybe_dl as (bar, sink):
+            content = element.content
+            content = await self.internalize_images(content)
+            sink.file.write(learning_module_template(content, maybe_dl.path.name, prev, next).encode("utf-8"))
+            sink.done()
+
+    async def internalize_images(self, tag: Tag) -> Tag:
+        """
+        Tries to fetch ILIAS images and embed them as base64 data.
+        """
+        log.explain_topic("Internalizing images")
+        for elem in tag.find_all(recursive=True):
+            if not isinstance(elem, Tag):
+                continue
+            if elem.name == "img":
+                if src := elem.attrs.get("src", None):
+                    url = urljoin(_ILIAS_URL, src)
+                    if not url.startswith(_ILIAS_URL):
+                        continue
+                    log.explain(f"Internalizing {url!r}")
+                    img = await self._get_authenticated(url)
+                    elem.attrs["src"] = "data:;base64," + base64.b64encode(img).decode()
+            if elem.name == "iframe" and elem.attrs.get("src", "").startswith("//"):
+                # For unknown reasons the protocol seems to be stripped.
+                elem.attrs["src"] = "https:" + elem.attrs["src"]
+        return tag
+
+    async def _get_page(self, url: str, root_page_allowed: bool = False) -> BeautifulSoup:
         auth_id = await self._current_auth_id()
         async with self.session.get(url) as request:
             soup = soupify(await request.read())
             if self._is_logged_in(soup):
-                return soup
+                return self._verify_page(soup, url, root_page_allowed)
 
         # We weren't authenticated, so try to do that
         await self.authenticate(auth_id)
@@ -428,21 +887,80 @@ class KitIliasWebCrawler(HttpCrawler):
         async with self.session.get(url) as request:
             soup = soupify(await request.read())
             if self._is_logged_in(soup):
-                return soup
+                return self._verify_page(soup, url, root_page_allowed)
         raise CrawlError("get_page failed even after authenticating")
 
+    def _verify_page(self, soup: BeautifulSoup, url: str, root_page_allowed: bool) -> BeautifulSoup:
+        if IliasPage.is_root_page(soup) and not root_page_allowed:
+            raise CrawlError(
+                "Unexpectedly encountered ILIAS root page. "
+                "This usually happens because the ILIAS instance is broken. "
+                "If so, wait a day or two and try again. "
+                "It could also happen because a crawled element links to the ILIAS root page. "
+                "If so, use a transform with a ! as target to ignore the particular element. "
+                f"The redirect came from {url}"
+            )
+        return soup
+
+    async def _post_authenticated(
+        self,
+        url: str,
+        data: dict[str, Union[str, List[str]]]
+    ) -> bytes:
+        auth_id = await self._current_auth_id()
+
+        form_data = aiohttp.FormData()
+        for key, val in data.items():
+            form_data.add_field(key, val)
+
+        async with self.session.post(url, data=form_data(), allow_redirects=False) as request:
+            if request.status == 200:
+                return await request.read()
+
+        # We weren't authenticated, so try to do that
+        await self.authenticate(auth_id)
+
+        # Retry once after authenticating. If this fails, we will die.
+        async with self.session.post(url, data=data, allow_redirects=False) as request:
+            if request.status == 200:
+                return await request.read()
+        raise CrawlError("post_authenticated failed even after authenticating")
+
+    async def _get_authenticated(self, url: str) -> bytes:
+        auth_id = await self._current_auth_id()
+
+        async with self.session.get(url, allow_redirects=False) as request:
+            if request.status == 200:
+                return await request.read()
+
+        # We weren't authenticated, so try to do that
+        await self.authenticate(auth_id)
+
+        # Retry once after authenticating. If this fails, we will die.
+        async with self.session.get(url, allow_redirects=False) as request:
+            if request.status == 200:
+                return await request.read()
+        raise CrawlError("get_authenticated failed even after authenticating")
+
     # We repeat this as the login method in shibboleth doesn't handle I/O errors.
     # Shibboleth is quite reliable as well, the repeat is likely not critical here.
-    @_iorepeat(3, "Login")
+    @ _iorepeat(3, "Login", failure_is_error=True)
     async def _authenticate(self) -> None:
         await self._shibboleth_login.login(self.session)
 
     @ staticmethod
     def _is_logged_in(soup: BeautifulSoup) -> bool:
         # Normal ILIAS pages
-        userlog = soup.find("li", {"id": "userlog"})
+        mainbar: Optional[Tag] = soup.find(class_="il-maincontrols-metabar")
-        if userlog is not None:
+        if mainbar is not None:
+            login_button = mainbar.find(attrs={"href": lambda x: x and "login.php" in x})
+            shib_login = soup.find(id="button_shib_login")
+            return not login_button and not shib_login
+
+        # Personal Desktop
+        if soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x}):
             return True
+
         # Video listing embeds do not have complete ILIAS html. Try to match them by
         # their video listing table
         video_table = soup.find(
@@ -480,14 +998,17 @@ class KitShibbolethLogin:
 
         # Equivalent: Click on "Mit KIT-Account anmelden" button in
         # https://ilias.studium.kit.edu/login.php
-        url = "https://ilias.studium.kit.edu/Shibboleth.sso/Login"
+        url = f"{_ILIAS_URL}/shib_login.php"
         data = {
             "sendLogin": "1",
             "idp_selection": "https://idp.scc.kit.edu/idp/shibboleth",
-            "target": "/shib_login.php",
+            "il_target": "",
-            "home_organization_selection": "Mit KIT-Account anmelden",
+            "home_organization_selection": "Weiter",
         }
-        soup: BeautifulSoup = await _post(sess, url, data)
+        soup: Union[BeautifulSoup, KitShibbolethBackgroundLoginSuccessful] = await _shib_post(sess, url, data)
+
+        if isinstance(soup, KitShibbolethBackgroundLoginSuccessful):
+            return
 
         # Attempt to login using credentials, if necessary
         while not self._login_successful(soup):
@@ -510,6 +1031,12 @@ class KitShibbolethLogin:
             }
             soup = await _post(sess, url, data)
 
+            if soup.find(id="attributeRelease"):
+                raise CrawlError(
+                    "ILIAS Shibboleth entitlements changed! "
+                    "Please log in once in your browser and review them"
+                )
+
             if self._tfa_required(soup):
                 soup = await self._authenticate_tfa(sess, soup)
 
@@ -520,7 +1047,7 @@ class KitShibbolethLogin:
         # (or clicking "Continue" if you have JS disabled)
         relay_state = soup.find("input", {"name": "RelayState"})
         saml_response = soup.find("input", {"name": "SAMLResponse"})
-        url = "https://ilias.studium.kit.edu/Shibboleth.sso/SAML2/POST"
+        url = f"{_ILIAS_URL}/Shibboleth.sso/SAML2/POST"
         data = {  # using the info obtained in the while loop above
             "RelayState": relay_state["value"],
             "SAMLResponse": saml_response["value"],
@@ -567,3 +1094,51 @@ class KitShibbolethLogin:
 async def _post(session: aiohttp.ClientSession, url: str, data: Any) -> BeautifulSoup:
     async with session.post(url, data=data) as response:
         return soupify(await response.read())
+
+
+async def _shib_post(
+    session: aiohttp.ClientSession,
+    url: str,
+    data: Any
+) -> Union[BeautifulSoup, KitShibbolethBackgroundLoginSuccessful]:
+    """
+    aiohttp unescapes '/' and ':' in URL query parameters which is not RFC compliant and rejected
+    by Shibboleth. Thanks a lot. So now we unroll the requests manually, parse location headers and
+    build encoded URL objects ourselves... Who thought mangling location header was a good idea??
+    """
+    log.explain_topic("Shib login POST")
+    async with session.post(url, data=data, allow_redirects=False) as response:
+        location = response.headers.get("location")
+        log.explain(f"Got location {location!r}")
+        if not location:
+            raise CrawlWarning(f"Login failed (1), no location header present at {url}")
+        correct_url = yarl.URL(location, encoded=True)
+        log.explain(f"Corrected location to {correct_url!r}")
+
+        if str(correct_url).startswith(_ILIAS_URL):
+            log.explain("ILIAS recognized our shib token and logged us in in the background, returning")
+            return KitShibbolethBackgroundLoginSuccessful()
+
+        async with session.get(correct_url, allow_redirects=False) as response:
+            location = response.headers.get("location")
+            log.explain(f"Redirected to {location!r} with status {response.status}")
+            # If shib still still has a valid session, it will directly respond to the request
+            if location is None:
+                log.explain("Shib recognized us, returning its response directly")
+                return soupify(await response.read())
+
+            as_yarl = yarl.URL(response.url)
+            # Probably not needed anymore, but might catch a few weird situations with a nicer message
+            if not location or not as_yarl.host:
+                raise CrawlWarning(f"Login failed (2), no location header present at {correct_url}")
+
+            correct_url = yarl.URL.build(
+                scheme=as_yarl.scheme,
+                host=as_yarl.host,
+                path=location,
+                encoded=True
+            )
+            log.explain(f"Corrected location to {correct_url!r}")
+
+            async with session.get(correct_url, allow_redirects=False) as response:
+                return soupify(await response.read())

PFERD/crawl/kit_ipd_crawler.py

@@ -0,0 +1,170 @@
+import os
+import re
+from dataclasses import dataclass
+from pathlib import PurePath
+from typing import Awaitable, List, Optional, Pattern, Set, Tuple, Union
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup, Tag
+
+from ..config import Config
+from ..logging import ProgressBar, log
+from ..output_dir import FileSink
+from ..utils import soupify
+from .crawler import CrawlError
+from .http_crawler import HttpCrawler, HttpCrawlerSection
+
+
+class KitIpdCrawlerSection(HttpCrawlerSection):
+    def target(self) -> str:
+        target = self.s.get("target")
+        if not target:
+            self.missing_value("target")
+
+        if not target.startswith("https://"):
+            self.invalid_value("target", target, "Should be a URL")
+
+        return target
+
+    def link_regex(self) -> Pattern[str]:
+        regex = self.s.get("link_regex", r"^.*?[^/]+\.(pdf|zip|c|cpp|java)$")
+        return re.compile(regex)
+
+
+@dataclass(unsafe_hash=True)
+class KitIpdFile:
+    name: str
+    url: str
+
+
+@dataclass
+class KitIpdFolder:
+    name: str
+    files: List[KitIpdFile]
+
+    def explain(self) -> None:
+        log.explain_topic(f"Folder {self.name!r}")
+        for file in self.files:
+            log.explain(f"File {file.name!r} (href={file.url!r})")
+
+    def __hash__(self) -> int:
+        return self.name.__hash__()
+
+
+class KitIpdCrawler(HttpCrawler):
+
+    def __init__(
+            self,
+            name: str,
+            section: KitIpdCrawlerSection,
+            config: Config,
+    ):
+        super().__init__(name, section, config)
+        self._url = section.target()
+        self._file_regex = section.link_regex()
+
+    async def _run(self) -> None:
+        maybe_cl = await self.crawl(PurePath("."))
+        if not maybe_cl:
+            return
+
+        tasks: List[Awaitable[None]] = []
+
+        async with maybe_cl:
+            for item in await self._fetch_items():
+                if isinstance(item, KitIpdFolder):
+                    tasks.append(self._crawl_folder(item))
+                else:
+                    # Orphan files are placed in the root folder
+                    tasks.append(self._download_file(PurePath("."), item))
+
+        await self.gather(tasks)
+
+    async def _crawl_folder(self, folder: KitIpdFolder) -> None:
+        path = PurePath(folder.name)
+        if not await self.crawl(path):
+            return
+
+        tasks = [self._download_file(path, file) for file in folder.files]
+
+        await self.gather(tasks)
+
+    async def _download_file(self, parent: PurePath, file: KitIpdFile) -> None:
+        element_path = parent / file.name
+        maybe_dl = await self.download(element_path)
+        if not maybe_dl:
+            return
+
+        async with maybe_dl as (bar, sink):
+            await self._stream_from_url(file.url, sink, bar)
+
+    async def _fetch_items(self) -> Set[Union[KitIpdFile, KitIpdFolder]]:
+        page, url = await self.get_page()
+        elements: List[Tag] = self._find_file_links(page)
+        items: Set[Union[KitIpdFile, KitIpdFolder]] = set()
+
+        for element in elements:
+            folder_label = self._find_folder_label(element)
+            if folder_label:
+                folder = self._extract_folder(folder_label, url)
+                if folder not in items:
+                    items.add(folder)
+                    folder.explain()
+            else:
+                file = self._extract_file(element, url)
+                items.add(file)
+                log.explain_topic(f"Orphan file {file.name!r} (href={file.url!r})")
+                log.explain("Attributing it to root folder")
+
+        return items
+
+    def _extract_folder(self, folder_tag: Tag, url: str) -> KitIpdFolder:
+        files: List[KitIpdFile] = []
+        name = folder_tag.getText().strip()
+
+        container: Tag = folder_tag.findNextSibling(name="table")
+        for link in self._find_file_links(container):
+            files.append(self._extract_file(link, url))
+
+        return KitIpdFolder(name, files)
+
+    @staticmethod
+    def _find_folder_label(file_link: Tag) -> Optional[Tag]:
+        enclosing_table: Tag = file_link.findParent(name="table")
+        if enclosing_table is None:
+            return None
+        return enclosing_table.findPreviousSibling(name=re.compile("^h[1-6]$"))
+
+    def _extract_file(self, link: Tag, url: str) -> KitIpdFile:
+        url = self._abs_url_from_link(url, link)
+        name = os.path.basename(url)
+        return KitIpdFile(name, url)
+
+    def _find_file_links(self, tag: Union[Tag, BeautifulSoup]) -> List[Tag]:
+        return tag.findAll(name="a", attrs={"href": self._file_regex})
+
+    def _abs_url_from_link(self, url: str, link_tag: Tag) -> str:
+        return urljoin(url, link_tag.get("href"))
+
+    async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar) -> None:
+        async with self.session.get(url, allow_redirects=False) as resp:
+            if resp.status == 403:
+                raise CrawlError("Received a 403. Are you within the KIT network/VPN?")
+            if resp.content_length:
+                bar.set_total(resp.content_length)
+
+            async for data in resp.content.iter_chunked(1024):
+                sink.file.write(data)
+                bar.advance(len(data))
+
+            sink.done()
+
+    async def get_page(self) -> Tuple[BeautifulSoup, str]:
+        async with self.session.get(self._url) as request:
+            # The web page for Algorithmen für Routenplanung contains some
+            # weird comments that beautifulsoup doesn't parse correctly. This
+            # hack enables those pages to be crawled, and should hopefully not
+            # cause issues on other pages.
+            content = (await request.read()).decode("utf-8")
+            content = re.sub(r"<!--.*?-->", "", content)
+            return soupify(content.encode("utf-8")), str(request.url)

PFERD/deduplicator.py

@@ -56,6 +56,12 @@ class Deduplicator:
             log.explain(f"Changed path to {fmt_path(new_path)} for windows compatibility")
         return new_path
 
+    def fixup_path(self, path: PurePath) -> PurePath:
+        """Fixes up the path for windows, if enabled. Returns the path unchanged otherwise."""
+        if self._windows_paths:
+            return self._fixup_for_windows(path)
+        return path
+
     def mark(self, path: PurePath) -> PurePath:
         if self._windows_paths:
             path = self._fixup_for_windows(path)

PFERD/logging.py

@@ -5,7 +5,7 @@ from contextlib import asynccontextmanager, contextmanager
 # TODO In Python 3.9 and above, ContextManager is deprecated
 from typing import AsyncIterator, ContextManager, Iterator, List, Optional
 
-from rich.console import Console, RenderGroup
+from rich.console import Console, Group
 from rich.live import Live
 from rich.markup import escape
 from rich.panel import Panel
@@ -59,6 +59,7 @@ class Log:
         # Whether different parts of the output are enabled or disabled
         self.output_explain = False
         self.output_status = True
+        self.output_not_deleted = True
         self.output_report = True
 
     def _update_live(self) -> None:
@@ -68,7 +69,7 @@ class Log:
         if self._download_progress.task_ids:
             elements.append(self._download_progress)
 
-        group = RenderGroup(*elements)  # type: ignore
+        group = Group(*elements)
         self._live.update(group)
 
     @contextmanager
@@ -197,7 +198,7 @@ directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
         if self.output_explain:
             self.print(f"  {escape(text)}")
 
-    def status(self, style: str, action: str, text: str) -> None:
+    def status(self, style: str, action: str, text: str, suffix: str = "") -> None:
         """
         Print a status update while crawling. Allows markup in the "style"
         argument which will be applied to the "action" string.
@@ -205,7 +206,18 @@ directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
 
         if self.output_status:
             action = escape(f"{action:<{self.STATUS_WIDTH}}")
-            self.print(f"{style}{action}[/] {escape(text)}")
+            self.print(f"{style}{action}[/] {escape(text)} {suffix}")
+
+    def not_deleted(self, style: str, action: str, text: str, suffix: str = "") -> None:
+        """
+        Print a message for a local only file that wasn't
+        deleted while crawling. Allows markup in the "style"
+        argument which will be applied to the "action" string.
+        """
+
+        if self.output_status and self.output_not_deleted:
+            action = escape(f"{action:<{self.STATUS_WIDTH}}")
+            self.print(f"{style}{action}[/] {escape(text)} {suffix}")
 
     def report(self, text: str) -> None:
         """
@@ -215,6 +227,14 @@ directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
         if self.output_report:
             self.print(text)
 
+    def report_not_deleted(self, text: str) -> None:
+        """
+        Print a report for a local only file that wasn't deleted after crawling. Allows markup.
+        """
+
+        if self.output_report and self.output_not_deleted:
+            self.print(text)
+
     @contextmanager
     def _bar(
             self,

PFERD/output_dir.py

@@ -44,6 +44,7 @@ class OnConflict(Enum):
     LOCAL_FIRST = "local-first"
     REMOTE_FIRST = "remote-first"
     NO_DELETE = "no-delete"
+    NO_DELETE_PROMPT_OVERWRITE = "no-delete-prompt-overwrite"
 
     @staticmethod
     def from_string(string: str) -> "OnConflict":
@@ -51,7 +52,7 @@ class OnConflict(Enum):
             return OnConflict(string)
         except ValueError:
             raise ValueError("must be one of 'prompt', 'local-first',"
-                             " 'remote-first', 'no-delete'")
+                             " 'remote-first', 'no-delete', 'no-delete-prompt-overwrite'")
 
 
 @dataclass
@@ -231,7 +232,10 @@ class OutputDirectory:
         stat = local_path.stat()
 
         remote_newer = None
-        if mtime := heuristics.mtime:
+
+        # Python on Windows crashes when faced with timestamps around the unix epoch
+        if heuristics.mtime and (os.name != "nt" or heuristics.mtime.year > 1970):
+            mtime = heuristics.mtime
             remote_newer = mtime.timestamp() > stat.st_mtime
             if remote_newer:
                 log.explain("Remote file seems to be newer")
@@ -261,7 +265,7 @@ class OutputDirectory:
             on_conflict: OnConflict,
             path: PurePath,
     ) -> bool:
-        if on_conflict == OnConflict.PROMPT:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
             async with log.exclusive_output():
                 prompt = f"Replace {fmt_path(path)} with remote file?"
                 return await prompt_yes_no(prompt, default=False)
@@ -280,7 +284,7 @@ class OutputDirectory:
             on_conflict: OnConflict,
             path: PurePath,
     ) -> bool:
-        if on_conflict == OnConflict.PROMPT:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
             async with log.exclusive_output():
                 prompt = f"Recursively delete {fmt_path(path)} and replace with remote file?"
                 return await prompt_yes_no(prompt, default=False)
@@ -300,7 +304,7 @@ class OutputDirectory:
             path: PurePath,
             parent: PurePath,
     ) -> bool:
-        if on_conflict == OnConflict.PROMPT:
+        if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
             async with log.exclusive_output():
                 prompt = f"Delete {fmt_path(parent)} so remote file {fmt_path(path)} can be downloaded?"
                 return await prompt_yes_no(prompt, default=False)
@@ -327,7 +331,7 @@ class OutputDirectory:
             return False
         elif on_conflict == OnConflict.REMOTE_FIRST:
             return True
-        elif on_conflict == OnConflict.NO_DELETE:
+        elif on_conflict in {OnConflict.NO_DELETE, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
             return False
 
         # This should never be reached
@@ -492,7 +496,7 @@ class OutputDirectory:
             except OSError:
                 pass
         else:
-            log.status("[bold bright_magenta]", "Not deleted", fmt_path(pure))
+            log.not_deleted("[bold bright_magenta]", "Not deleted", fmt_path(pure))
             self._report.not_delete_file(pure)
 
     def load_prev_report(self) -> None:
@@ -500,7 +504,7 @@ class OutputDirectory:
         try:
             self._prev_report = Report.load(self._report_path)
             log.explain("Loaded report successfully")
-        except (OSError, json.JSONDecodeError, ReportLoadError) as e:
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError, ReportLoadError) as e:
             log.explain("Failed to load report")
             log.explain(str(e))
 

PFERD/pferd.py

@@ -1,11 +1,11 @@
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Set
 
 from rich.markup import escape
 
-from .auth import AUTHENTICATORS, Authenticator, AuthError
+from .auth import AUTHENTICATORS, Authenticator, AuthError, AuthSection
 from .config import Config, ConfigOptionError
-from .crawl import CRAWLERS, Crawler, CrawlError, KitIliasWebCrawler
+from .crawl import CRAWLERS, Crawler, CrawlError, CrawlerSection, KitIliasWebCrawler
 from .logging import log
 from .utils import fmt_path
 
@@ -15,41 +15,52 @@ class PferdLoadError(Exception):
 
 
 class Pferd:
-    def __init__(self, config: Config, cli_crawlers: Optional[List[str]]):
+    def __init__(self, config: Config, cli_crawlers: Optional[List[str]], cli_skips: Optional[List[str]]):
         """
         May throw PferdLoadError.
         """
 
         self._config = config
-        self._crawlers_to_run = self._find_crawlers_to_run(config, cli_crawlers)
+        self._crawlers_to_run = self._find_crawlers_to_run(config, cli_crawlers, cli_skips)
 
         self._authenticators: Dict[str, Authenticator] = {}
         self._crawlers: Dict[str, Crawler] = {}
 
-    def _find_crawlers_to_run(self, config: Config, cli_crawlers: Optional[List[str]]) -> List[str]:
+    def _find_config_crawlers(self, config: Config) -> List[str]:
-        log.explain_topic("Deciding which crawlers to run")
+        crawl_sections = []
-        crawl_sections = [name for name, _ in config.crawl_sections()]
+
+        for name, section in config.crawl_sections():
+            if CrawlerSection(section).skip():
+                log.explain(f"Skipping {name!r}")
+            else:
+                crawl_sections.append(name)
 
-        if cli_crawlers is None:
-            log.explain("No crawlers specified on CLI")
-            log.explain("Running all crawlers specified in config")
         return crawl_sections
 
+    def _find_cli_crawlers(self, config: Config, cli_crawlers: List[str]) -> List[str]:
         if len(cli_crawlers) != len(set(cli_crawlers)):
             raise PferdLoadError("Some crawlers were selected multiple times")
 
-        log.explain("Crawlers specified on CLI")
+        crawl_sections = [name for name, _ in config.crawl_sections()]
 
-        crawlers_to_run = []  # With crawl: prefix
+        crawlers_to_run = set()  # With crawl: prefix
         unknown_names = []  # Without crawl: prefix
 
         for name in cli_crawlers:
             section_name = f"crawl:{name}"
             if section_name in crawl_sections:
                 log.explain(f"Crawler section named {section_name!r} exists")
-                crawlers_to_run.append(section_name)
+                crawlers_to_run.add(section_name)
-            else:
+            # interprete name as alias of a crawler
-                log.explain(f"There's no crawler section named {section_name!r}")
+            alias_names = self._find_crawlers_by_alias(name, config)
+            if alias_names:
+                crawlers_to_run.update(alias_names)
+                log.explain_topic(f"Crawler alias {name!r} found corresponding crawler sections:")
+                for alias_name in alias_names:
+                    log.explain(f"Crawler section named {alias_name!r} with alias {name!r} exists")
+
+            if not section_name in crawl_sections and not alias_names:
+                log.explain(f"There's neither a crawler section named {section_name!r} nor does a crawler with alias {name!r} exist.")
                 unknown_names.append(name)
 
         if unknown_names:
@@ -62,10 +73,44 @@ class Pferd:
 
         return crawlers_to_run
 
+    def _find_crawlers_by_alias(self, alias: str, config: Config) -> Set[str]:
+        alias_names = set()
+        for (section_name, section) in config.crawl_sections():
+            section_aliases = section.get("aliases", [])
+            if alias in section_aliases:
+                alias_names.add(section_name)
+        return alias_names
+
+    def _find_crawlers_to_run(
+            self,
+            config: Config,
+            cli_crawlers: Optional[List[str]],
+            cli_skips: Optional[List[str]],
+    ) -> List[str]:
+        log.explain_topic("Deciding which crawlers to run")
+
+        crawlers: List[str]
+        if cli_crawlers is None:
+            log.explain("No crawlers specified on CLI")
+            log.explain("Running crawlers specified in config")
+            crawlers = self._find_config_crawlers(config)
+        else:
+            log.explain("Crawlers specified on CLI")
+            crawlers = self._find_cli_crawlers(config, cli_crawlers)
+
+        skips = {f"crawl:{name}" for name in cli_skips} if cli_skips else set()
+        for crawler in crawlers:
+            if crawler in skips:
+                log.explain(f"Skipping crawler {crawler!r}")
+        crawlers = [crawler for crawler in crawlers if crawler not in skips]
+
+        return crawlers
+
     def _load_authenticators(self) -> None:
         for name, section in self._config.auth_sections():
             log.print(f"[bold bright_cyan]Loading[/] {escape(name)}")
-            auth_type = section.get("type")
+
+            auth_type = AuthSection(section).type()
             authenticator_constructor = AUTHENTICATORS.get(auth_type)
             if authenticator_constructor is None:
                 raise ConfigOptionError(name, "type", f"Unknown authenticator type: {auth_type!r}")
@@ -80,7 +125,7 @@ class Pferd:
         for name, section in self._config.crawl_sections():
             log.print(f"[bold bright_cyan]Loading[/] {escape(name)}")
 
-            crawl_type = section.get("type")
+            crawl_type = CrawlerSection(section).type()
             crawler_constructor = CRAWLERS.get(crawl_type)
             if crawler_constructor is None:
                 raise ConfigOptionError(name, "type", f"Unknown crawler type: {crawl_type!r}")
@@ -151,7 +196,15 @@ class Pferd:
                 log.report(f"  [bold bright_magenta]Deleted[/] {fmt_path(path)}")
             for path in sorted(crawler.report.not_deleted_files):
                 something_changed = True
-                log.report(f"  [bold bright_magenta]Not deleted[/] {fmt_path(path)}")
+                log.report_not_deleted(f"  [bold bright_magenta]Not deleted[/] {fmt_path(path)}")
+
+            for warning in crawler.report.encountered_warnings:
+                something_changed = True
+                log.report(f"  [bold bright_red]Warning[/] {warning}")
+
+            for error in crawler.report.encountered_errors:
+                something_changed = True
+                log.report(f"  [bold bright_red]Error[/] {error}")
 
             if not something_changed:
                 log.report("  Nothing changed")

PFERD/report.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path, PurePath
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, List, Optional, Set
 
 
 class ReportLoadError(Exception):
@@ -68,6 +68,13 @@ class Report:
         # Files that should have been deleted by the cleanup but weren't
         self.not_deleted_files: Set[PurePath] = set()
 
+        # Custom crawler-specific data
+        self.custom: Dict[str, Any] = dict()
+
+        # Encountered errors and warnings
+        self.encountered_warnings: List[str] = []
+        self.encountered_errors: List[str] = []
+
     @staticmethod
     def _get_list_of_strs(data: Dict[str, Any], key: str) -> List[str]:
         result: Any = data.get(key, [])
@@ -81,13 +88,22 @@ class Report:
 
         return result
 
+    @staticmethod
+    def _get_str_dictionary(data: Dict[str, Any], key: str) -> Dict[str, Any]:
+        result: Dict[str, Any] = data.get(key, {})
+
+        if not isinstance(result, dict):
+            raise ReportLoadError(f"Incorrect format: {key!r} is not a dictionary")
+
+        return result
+
     @classmethod
     def load(cls, path: Path) -> "Report":
         """
-        May raise OSError, JsonDecodeError, ReportLoadError.
+        May raise OSError, UnicodeDecodeError, JsonDecodeError, ReportLoadError.
         """
 
-        with open(path) as f:
+        with open(path, encoding="utf-8") as f:
             data = json.load(f)
 
         if not isinstance(data, dict):
@@ -108,6 +124,9 @@ class Report:
             self.delete_file(PurePath(elem))
         for elem in self._get_list_of_strs(data, "not_deleted"):
             self.not_delete_file(PurePath(elem))
+        self.custom = self._get_str_dictionary(data, "custom")
+        self.encountered_errors = self._get_list_of_strs(data, "encountered_errors")
+        self.encountered_warnings = self._get_list_of_strs(data, "encountered_warnings")
 
         return self
 
@@ -124,9 +143,12 @@ class Report:
             "changed": [str(path) for path in sorted(self.changed_files)],
             "deleted": [str(path) for path in sorted(self.deleted_files)],
             "not_deleted": [str(path) for path in sorted(self.not_deleted_files)],
+            "custom": self.custom,
+            "encountered_warnings": self.encountered_warnings,
+            "encountered_errors": self.encountered_errors,
         }
 
-        with open(path, "w") as f:
+        with open(path, "w", encoding="utf-8") as f:
             json.dump(data, f, indent=2, sort_keys=True)
             f.write("\n")  # json.dump doesn't do this
 
@@ -190,3 +212,27 @@ class Report:
         """
 
         self.not_deleted_files.add(path)
+
+    def add_custom_value(self, key: str, value: Any) -> None:
+        """
+        Adds a custom value under the passed key, overwriting any existing
+        """
+        self.custom[key] = value
+
+    def get_custom_value(self, key: str) -> Optional[Any]:
+        """
+        Retrieves a custom value for the given key.
+        """
+        return self.custom.get(key)
+
+    def add_error(self, error: str) -> None:
+        """
+        Adds an error to this report's error list.
+        """
+        self.encountered_errors.append(error)
+
+    def add_warning(self, warning: str) -> None:
+        """
+        Adds a warning to this report's warning list.
+        """
+        self.encountered_warnings.append(warning)

PFERD/transformer.py

@@ -1,151 +1,166 @@
-# I'm sorry that this code has become a bit dense and unreadable. While
-# reading, it is important to remember what True and False mean. I'd love to
-# have some proper sum-types for the inputs and outputs, they'd make this code
-# a lot easier to understand.
-
 import ast
 import re
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
 from pathlib import PurePath
-from typing import Dict, Optional, Sequence, Union
+from typing import Callable, Dict, List, Optional, Sequence, TypeVar, Union
 
 from .logging import log
-from .utils import fmt_path
+from .utils import fmt_path, str_path
 
 
-class Rule(ABC):
+class ArrowHead(Enum):
-    @abstractmethod
+    NORMAL = 0
-    def transform(self, path: PurePath) -> Union[PurePath, bool]:
+    SEQUENCE = 1
-        """
-        Try to apply this rule to the path. Returns another path if the rule
-        was successfully applied, True if the rule matched but resulted in an
-        exclamation mark, and False if the rule didn't match at all.
-        """
 
+
+class Ignore:
     pass
 
 
-# These rules all use a Union[T, bool] for their right side. They are passed a
+class Empty:
-# T if the arrow's right side was a normal string, True if it was an
+    pass
-# exclamation mark and False if it was missing entirely.
 
-class NormalRule(Rule):
-    def __init__(self, left: PurePath, right: Union[PurePath, bool]):
 
-        self._left = left
+RightSide = Union[str, Ignore, Empty]
-        self._right = right
 
-    def _match_prefix(self, path: PurePath) -> Optional[PurePath]:
-        left_parts = list(reversed(self._left.parts))
-        path_parts = list(reversed(path.parts))
 
-        if len(left_parts) > len(path_parts):
+@dataclass
+class Transformed:
+    path: PurePath
+
+
+class Ignored:
+    pass
+
+
+TransformResult = Optional[Union[Transformed, Ignored]]
+
+
+@dataclass
+class Rule:
+    left: str
+    left_index: int
+    name: str
+    head: ArrowHead
+    right: RightSide
+    right_index: int
+
+    def right_result(self, path: PurePath) -> Union[str, Transformed, Ignored]:
+        if isinstance(self.right, str):
+            return self.right
+        elif isinstance(self.right, Ignore):
+            return Ignored()
+        elif isinstance(self.right, Empty):
+            return Transformed(path)
+        else:
+            raise RuntimeError(f"Right side has invalid type {type(self.right)}")
+
+
+class Transformation(ABC):
+    def __init__(self, rule: Rule):
+        self.rule = rule
+
+    @abstractmethod
+    def transform(self, path: PurePath) -> TransformResult:
+        pass
+
+
+class ExactTf(Transformation):
+    def transform(self, path: PurePath) -> TransformResult:
+        if path != PurePath(self.rule.left):
             return None
 
-        while left_parts and path_parts:
+        right = self.rule.right_result(path)
-            left_part = left_parts.pop()
+        if not isinstance(right, str):
-            path_part = path_parts.pop()
+            return right
 
-            if left_part != path_part:
+        return Transformed(PurePath(right))
+
+
+class ExactReTf(Transformation):
+    def transform(self, path: PurePath) -> TransformResult:
+        match = re.fullmatch(self.rule.left, str_path(path))
+        if not match:
             return None
 
-        if left_parts:
+        right = self.rule.right_result(path)
-            return None
+        if not isinstance(right, str):
+            return right
 
-        path_parts.reverse()
+        # For some reason, mypy thinks that "groups" has type List[str]. But
-        return PurePath(*path_parts)
+        # since elements of "match.groups()" can be None, mypy is wrong.
-
-    def transform(self, path: PurePath) -> Union[PurePath, bool]:
-        if rest := self._match_prefix(path):
-            if isinstance(self._right, bool):
-                return self._right or path
-            else:
-                return self._right / rest
-
-        return False
-
-
-class ExactRule(Rule):
-    def __init__(self, left: PurePath, right: Union[PurePath, bool]):
-        self._left = left
-        self._right = right
-
-    def transform(self, path: PurePath) -> Union[PurePath, bool]:
-        if path == self._left:
-            if isinstance(self._right, bool):
-                return self._right or path
-            else:
-                return self._right
-
-        return False
-
-
-class NameRule(Rule):
-    def __init__(self, subrule: Rule):
-        self._subrule = subrule
-
-    def transform(self, path: PurePath) -> Union[PurePath, bool]:
-        matched = False
-        result = PurePath()
-
-        for part in path.parts:
-            part_result = self._subrule.transform(PurePath(part))
-            if isinstance(part_result, PurePath):
-                matched = True
-                result /= part_result
-            elif part_result:
-                # If any subrule call ignores its path segment, the entire path
-                # should be ignored
-                return True
-            else:
-                # The subrule doesn't modify this segment, but maybe other
-                # segments
-                result /= part
-
-        if matched:
-            return result
-        else:
-            # The subrule has modified no segments, so this name version of it
-            # doesn't match
-            return False
-
-
-class ReRule(Rule):
-    def __init__(self, left: str, right: Union[str, bool]):
-        self._left = left
-        self._right = right
-
-    def transform(self, path: PurePath) -> Union[PurePath, bool]:
-        if match := re.fullmatch(self._left, str(path)):
-            if isinstance(self._right, bool):
-                return self._right or path
-
-            vars: Dict[str, Union[str, int, float]] = {}
-
-            # For some reason, mypy thinks that "groups" has type List[str].
-            # But since elements of "match.groups()" can be None, mypy is
-            # wrong.
         groups: Sequence[Optional[str]] = [match[0]] + list(match.groups())
+
+        locals_dir: Dict[str, Union[str, int, float]] = {}
         for i, group in enumerate(groups):
             if group is None:
                 continue
 
-                vars[f"g{i}"] = group
+            locals_dir[f"g{i}"] = group
 
             try:
-                    vars[f"i{i}"] = int(group)
+                locals_dir[f"i{i}"] = int(group)
             except ValueError:
                 pass
 
             try:
-                    vars[f"f{i}"] = float(group)
+                locals_dir[f"f{i}"] = float(group)
             except ValueError:
                 pass
 
-            result = eval(f"f{self._right!r}", vars)
+        result = eval(f"f{right!r}", {}, locals_dir)
-            return PurePath(result)
+        return Transformed(PurePath(result))
 
-        return False
+
+class RenamingParentsTf(Transformation):
+    def __init__(self, sub_tf: Transformation):
+        super().__init__(sub_tf.rule)
+        self.sub_tf = sub_tf
+
+    def transform(self, path: PurePath) -> TransformResult:
+        for i in range(len(path.parts), -1, -1):
+            parent = PurePath(*path.parts[:i])
+            child = PurePath(*path.parts[i:])
+
+            transformed = self.sub_tf.transform(parent)
+            if not transformed:
+                continue
+            elif isinstance(transformed, Transformed):
+                return Transformed(transformed.path / child)
+            elif isinstance(transformed, Ignored):
+                return transformed
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(transformed)}: {transformed}")
+
+        return None
+
+
+class RenamingPartsTf(Transformation):
+    def __init__(self, sub_tf: Transformation):
+        super().__init__(sub_tf.rule)
+        self.sub_tf = sub_tf
+
+    def transform(self, path: PurePath) -> TransformResult:
+        result = PurePath()
+        any_part_matched = False
+        for part in path.parts:
+            transformed = self.sub_tf.transform(PurePath(part))
+            if not transformed:
+                result /= part
+            elif isinstance(transformed, Transformed):
+                result /= transformed.path
+                any_part_matched = True
+            elif isinstance(transformed, Ignored):
+                return transformed
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(transformed)}: {transformed}")
+
+        if any_part_matched:
+            return Transformed(result)
+        else:
+            return None
 
 
 class RuleParseError(Exception):
@@ -162,18 +177,15 @@ class RuleParseError(Exception):
         log.error_contd(f"{spaces}^--- {self.reason}")
 
 
+T = TypeVar("T")
+
+
 class Line:
     def __init__(self, line: str, line_nr: int):
         self._line = line
         self._line_nr = line_nr
         self._index = 0
 
-    def get(self) -> Optional[str]:
-        if self._index < len(self._line):
-            return self._line[self._index]
-
-        return None
-
     @property
     def line(self) -> str:
         return self._line
@@ -190,155 +202,196 @@ class Line:
     def index(self, index: int) -> None:
         self._index = index
 
-    def advance(self) -> None:
+    @property
-        self._index += 1
+    def rest(self) -> str:
+        return self.line[self.index:]
 
-    def expect(self, string: str) -> None:
+    def peek(self, amount: int = 1) -> str:
-        for char in string:
+        return self.rest[:amount]
-            if self.get() == char:
+
-                self.advance()
+    def take(self, amount: int = 1) -> str:
+        string = self.peek(amount)
+        self.index += len(string)
+        return string
+
+    def expect(self, string: str) -> str:
+        if self.peek(len(string)) == string:
+            return self.take(len(string))
         else:
-                raise RuleParseError(self, f"Expected {char!r}")
+            raise RuleParseError(self, f"Expected {string!r}")
+
+    def expect_with(self, string: str, value: T) -> T:
+        self.expect(string)
+        return value
+
+    def one_of(self, parsers: List[Callable[[], T]], description: str) -> T:
+        for parser in parsers:
+            index = self.index
+            try:
+                return parser()
+            except RuleParseError:
+                self.index = index
+
+        raise RuleParseError(self, description)
+
+
+# RULE = LEFT SPACE '-' NAME '-' HEAD (SPACE RIGHT)?
+# SPACE = ' '+
+# NAME = '' | 'exact' | 'name' | 're' | 'exact-re' | 'name-re'
+# HEAD = '>' | '>>'
+# LEFT = STR | QUOTED_STR
+# RIGHT = STR | QUOTED_STR | '!'
+
+
+def parse_zero_or_more_spaces(line: Line) -> None:
+    while line.peek() == " ":
+        line.take()
+
+
+def parse_one_or_more_spaces(line: Line) -> None:
+    line.expect(" ")
+    parse_zero_or_more_spaces(line)
+
+
+def parse_str(line: Line) -> str:
+    result = []
+    while c := line.peek():
+        if c == " ":
+            break
+        else:
+            line.take()
+            result.append(c)
+
+    if result:
+        return "".join(result)
+    else:
+        raise RuleParseError(line, "Expected non-space character")
 
 
 QUOTATION_MARKS = {'"', "'"}
 
 
-def parse_string_literal(line: Line) -> str:
+def parse_quoted_str(line: Line) -> str:
     escaped = False
 
     # Points to first character of string literal
     start_index = line.index
 
-    quotation_mark = line.get()
+    quotation_mark = line.peek()
     if quotation_mark not in QUOTATION_MARKS:
-        # This should never happen as long as this function is only called from
+        raise RuleParseError(line, "Expected quotation mark")
-        # parse_string.
+    line.take()
-        raise RuleParseError(line, "Invalid quotation mark")
-    line.advance()
 
-    while c := line.get():
+    while c := line.peek():
         if escaped:
             escaped = False
-            line.advance()
+            line.take()
         elif c == quotation_mark:
-            line.advance()
+            line.take()
             stop_index = line.index
             literal = line.line[start_index:stop_index]
+            try:
                 return ast.literal_eval(literal)
+            except SyntaxError as e:
+                line.index = start_index
+                raise RuleParseError(line, str(e)) from e
         elif c == "\\":
             escaped = True
-            line.advance()
+            line.take()
         else:
-            line.advance()
+            line.take()
 
     raise RuleParseError(line, "Expected end of string literal")
 
 
-def parse_until_space_or_eol(line: Line) -> str:
+def parse_left(line: Line) -> str:
-    result = []
+    if line.peek() in QUOTATION_MARKS:
-    while c := line.get():
+        return parse_quoted_str(line)
-        if c == " ":
-            break
-        result.append(c)
-        line.advance()
-
-    return "".join(result)
-
-
-def parse_string(line: Line) -> Union[str, bool]:
-    if line.get() in QUOTATION_MARKS:
-        return parse_string_literal(line)
     else:
-        string = parse_until_space_or_eol(line)
+        return parse_str(line)
+
+
+def parse_right(line: Line) -> Union[str, Ignore]:
+    c = line.peek()
+    if c in QUOTATION_MARKS:
+        return parse_quoted_str(line)
+    else:
+        string = parse_str(line)
         if string == "!":
-            return True
+            return Ignore()
         return string
 
 
-def parse_arrow(line: Line) -> str:
+def parse_arrow_name(line: Line) -> str:
-    line.expect("-")
+    return line.one_of([
-
+        lambda: line.expect("exact-re"),
-    name = []
+        lambda: line.expect("exact"),
-    while True:
+        lambda: line.expect("name-re"),
-        c = line.get()
+        lambda: line.expect("name"),
-        if not c:
+        lambda: line.expect("re"),
-            raise RuleParseError(line, "Expected rest of arrow")
+        lambda: line.expect(""),
-        elif c == "-":
+    ], "Expected arrow name")
-            line.advance()
-            c = line.get()
-            if not c:
-                raise RuleParseError(line, "Expected rest of arrow")
-            elif c == ">":
-                line.advance()
-                break  # End of arrow
-            else:
-                name.append("-")
-                continue
-        else:
-            name.append(c)
-
-        line.advance()
-
-    return "".join(name)
 
 
-def parse_whitespace(line: Line) -> None:
+def parse_arrow_head(line: Line) -> ArrowHead:
-    line.expect(" ")
+    return line.one_of([
-    while line.get() == " ":
+        lambda: line.expect_with(">>", ArrowHead.SEQUENCE),
-        line.advance()
+        lambda: line.expect_with(">", ArrowHead.NORMAL),
+    ], "Expected arrow head")
 
 
 def parse_eol(line: Line) -> None:
-    if line.get() is not None:
+    if line.peek():
         raise RuleParseError(line, "Expected end of line")
 
 
 def parse_rule(line: Line) -> Rule:
-    # Parse left side
+    parse_zero_or_more_spaces(line)
-    leftindex = line.index
+    left_index = line.index
-    left = parse_string(line)
+    left = parse_left(line)
-    if isinstance(left, bool):
-        line.index = leftindex
-        raise RuleParseError(line, "Left side can't be '!'")
-    leftpath = PurePath(left)
 
-    # Parse arrow
+    parse_one_or_more_spaces(line)
-    parse_whitespace(line)
-    arrowindex = line.index
-    arrowname = parse_arrow(line)
 
-    # Parse right side
+    line.expect("-")
-    if line.get():
+    name = parse_arrow_name(line)
-        parse_whitespace(line)
+    line.expect("-")
-        right = parse_string(line)
+    head = parse_arrow_head(line)
-    else:
-        right = False
-    rightpath: Union[PurePath, bool]
-    if isinstance(right, bool):
-        rightpath = right
-    else:
-        rightpath = PurePath(right)
 
+    right_index = line.index
+    right: RightSide
+    try:
+        parse_zero_or_more_spaces(line)
+        parse_eol(line)
+        right = Empty()
+    except RuleParseError:
+        line.index = right_index
+        parse_one_or_more_spaces(line)
+        right = parse_right(line)
         parse_eol(line)
 
-    # Dispatch
+    return Rule(left, left_index, name, head, right, right_index)
-    if arrowname == "":
+
-        return NormalRule(leftpath, rightpath)
+
-    elif arrowname == "name":
+def parse_transformation(line: Line) -> Transformation:
-        if len(leftpath.parts) > 1:
+    rule = parse_rule(line)
-            line.index = leftindex
+
-            raise RuleParseError(line, "SOURCE must be a single name, not multiple segments")
+    if rule.name == "":
-        return NameRule(ExactRule(leftpath, rightpath))
+        return RenamingParentsTf(ExactTf(rule))
-    elif arrowname == "exact":
+    elif rule.name == "exact":
-        return ExactRule(leftpath, rightpath)
+        return ExactTf(rule)
-    elif arrowname == "re":
+    elif rule.name == "name":
-        return ReRule(left, right)
+        if len(PurePath(rule.left).parts) > 1:
-    elif arrowname == "name-re":
+            line.index = rule.left_index
-        return NameRule(ReRule(left, right))
+            raise RuleParseError(line, "Expected name, not multiple segments")
+        return RenamingPartsTf(ExactTf(rule))
+    elif rule.name == "re":
+        return RenamingParentsTf(ExactReTf(rule))
+    elif rule.name == "exact-re":
+        return ExactReTf(rule)
+    elif rule.name == "name-re":
+        return RenamingPartsTf(ExactReTf(rule))
     else:
-        line.index = arrowindex + 1  # For nicer error message
+        raise RuntimeError(f"Invalid arrow name {rule.name!r}")
-        raise RuleParseError(line, f"Invalid arrow name {arrowname!r}")
 
 
 class Transformer:
@@ -347,32 +400,40 @@ class Transformer:
         May throw a RuleParseException.
         """
 
-        self._rules = []
+        self._tfs = []
         for i, line in enumerate(rules.split("\n")):
             line = line.strip()
             if line:
-                rule = parse_rule(Line(line, i))
+                tf = parse_transformation(Line(line, i))
-                self._rules.append((line, rule))
+                self._tfs.append((line, tf))
 
     def transform(self, path: PurePath) -> Optional[PurePath]:
-        for i, (line, rule) in enumerate(self._rules):
+        for i, (line, tf) in enumerate(self._tfs):
             log.explain(f"Testing rule {i+1}: {line}")
 
             try:
-                result = rule.transform(path)
+                result = tf.transform(path)
             except Exception as e:
                 log.warn(f"Error while testing rule {i+1}: {line}")
                 log.warn_contd(str(e))
                 continue
 
-            if isinstance(result, PurePath):
+            if not result:
-                log.explain(f"Match found, transformed path to {fmt_path(result)}")
-                return result
-            elif result:  # Exclamation mark
-                log.explain("Match found, path ignored")
-                return None
-            else:
                 continue
 
-        log.explain("No rule matched, path is unchanged")
+            if isinstance(result, Ignored):
+                log.explain("Match found, path ignored")
+                return None
+
+            if tf.rule.head == ArrowHead.NORMAL:
+                log.explain(f"Match found, transformed path to {fmt_path(result.path)}")
+                path = result.path
+                break
+            elif tf.rule.head == ArrowHead.SEQUENCE:
+                log.explain(f"Match found, updated path to {fmt_path(result.path)}")
+                path = result.path
+            else:
+                raise RuntimeError(f"Invalid transform result of type {type(result)}: {result}")
+
+        log.explain(f"Final result: {fmt_path(path)}")
         return path

PFERD/utils.py

@@ -91,8 +91,14 @@ def url_set_query_params(url: str, params: Dict[str, str]) -> str:
     return result
 
 
+def str_path(path: PurePath) -> str:
+    if not path.parts:
+        return "."
+    return "/".join(path.parts)
+
+
 def fmt_path(path: PurePath) -> str:
-    return repr(str(path))
+    return repr(str_path(path))
 
 
 def fmt_real_path(path: Path) -> str:

PFERD/version.py

@@ -1,2 +1,2 @@
 NAME = "PFERD"
-VERSION = "3.0.0"
+VERSION = "3.4.3"

README.md

@@ -17,7 +17,7 @@ Binaries for Linux, Windows and Mac can be downloaded directly from the
 
 ### With pip
 
-Ensure you have at least Python 3.8 installed. Run the following command to
+Ensure you have at least Python 3.9 installed. Run the following command to
 install PFERD or upgrade it to the latest version:
 
 ```
@@ -26,11 +26,22 @@ $ pip install --upgrade git+https://github.com/Garmelon/PFERD@latest
 
 The use of [venv](https://docs.python.org/3/library/venv.html) is recommended.
 
+### With package managers
+
+Unofficial packages are available for:
+- [AUR](https://aur.archlinux.org/packages/pferd)
+- [brew](https://formulae.brew.sh/formula/pferd)
+- [conda-forge](https://github.com/conda-forge/pferd-feedstock)
+- [nixpkgs](https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/misc/pferd/default.nix)
+- [PyPi](https://pypi.org/project/pferd)
+
+See also PFERD's [repology page](https://repology.org/project/pferd/versions).
+
 ## Basic usage
 
-PFERD can be run directly from the command line with no config file.
+PFERD can be run directly from the command line with no config file. Run `pferd
-Run `pferd -h` to get an overview of available commands and options.
+-h` to get an overview of available commands and options. Run `pferd <command>
-Run `pferd <command> -h` to see which options a command has.
+-h` to see which options a command has.
 
 For example, you can download your personal desktop from the KIT ILIAS like
 this:
@@ -116,17 +127,18 @@ transform =
   Online-Tests --> !
   Vorlesungswerbung --> !
 
+  # Rename folders
+  Lehrbücher --> Vorlesung
+  # Note the ">>" arrow head which lets us apply further rules to files moved to "Übung"
+  Übungsunterlagen -->> Übung
+
   # Move exercises to own folder. Rename them to "Blatt-XX.pdf" to make them sort properly
-  "Übungsunterlagen/(\d+). Übungsblatt.pdf" -re-> Blätter/Blatt-{i1:02}.pdf
+  "Übung/(\d+). Übungsblatt.pdf" -re-> Blätter/Blatt-{i1:02}.pdf
   # Move solutions to own folder. Rename them to "Blatt-XX-Lösung.pdf" to make them sort properly
-  "Übungsunterlagen/(\d+). Übungsblatt.*Musterlösung.pdf" -re-> Blätter/Blatt-{i1:02}-Lösung.pdf
+  "Übung/(\d+). Übungsblatt.*Musterlösung.pdf" -re-> Blätter/Blatt-{i1:02}-Lösung.pdf
 
   # The course has nested folders with the same name - flatten them
-  "Übungsunterlagen/(.+?)/\\1/(.*)" -re-> Übung/{g1}/{g2}
+  "Übung/(.+?)/\\1" -re-> Übung/{g1}
-
-  # Rename remaining folders
-  Übungsunterlagen --> Übung
-  Lehrbücher --> Vorlesung
 
 [crawl:Bar]
 type = kit-ilias-web

scripts/setup

@@ -12,6 +12,6 @@ pip install --upgrade setuptools
 # Installing PFERD itself
 pip install --editable .
 
-# Installing various tools
+# Installing tools and type hints
-pip install --upgrade mypy flake8 autopep8 isort
+pip install --upgrade mypy flake8 autopep8 isort pyinstaller
-pip install --upgrade pyinstaller
+pip install --upgrade types-chardet types-certifi

setup.cfg

@@ -4,13 +4,13 @@ version = attr: PFERD.version.VERSION
 
 [options]
 packages = find:
-python_requires = >=3.8
+python_requires = >=3.9
 install_requires =
-  aiohttp>=3.7.4.post0
+  aiohttp>=3.8.1
-  beautifulsoup4>=4.9.3
+  beautifulsoup4>=4.10.0
-  rich>=10.1.0
+  rich>=11.0.0
-  keyring>=23.0.1
+  keyring>=23.5.0
-  certifi>=2020.12.5
+  certifi>=2021.10.8
 
 [options.entry_points]
 console_scripts =