mirror of
https://github.com/Garmelon/PFERD.git
synced 2025-07-12 14:12:30 +02:00
Compare commits
10 Commits
debug/wind
...
sequential
| Author | SHA1 | Date | |
|---|---|---|---|
| bf27f4a686 | |||
| 5adfdfbd2b | |||
| 5c3942a13d | |||
| 5c9209b12e | |||
| 50c7778d38 | |||
| 354a22d1e3 | |||
| 6f87c5c774 | |||
| 1ca10571f0 | |||
| 10e1a5e871 | |||
| a2ffce4702 |
10
.github/dependabot.yml
vendored
10
.github/dependabot.yml
vendored
@ -1,10 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: monthly
|
||||
groups:
|
||||
gh-actions:
|
||||
patterns:
|
||||
- "*"
|
||||
29
.github/workflows/build-and-release.yml
vendored
29
.github/workflows/build-and-release.yml
vendored
@ -1,6 +1,6 @@
|
||||
name: build-and-release
|
||||
|
||||
on: [push, pull_request]
|
||||
on: push
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@ -13,12 +13,13 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13, macos-latest]
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
python: ["3.9"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
@ -33,12 +34,7 @@ jobs:
|
||||
run: ./scripts/setup --no-pip
|
||||
|
||||
- name: Run checks
|
||||
run: |
|
||||
./scripts/check
|
||||
./scripts/format
|
||||
|
||||
- name: Assert no changes
|
||||
run: git diff --exit-code
|
||||
run: ./scripts/check
|
||||
|
||||
- name: Build
|
||||
run: ./scripts/build
|
||||
@ -49,9 +45,9 @@ jobs:
|
||||
run: mv dist/pferd* dist/pferd-${{ matrix.os }}
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: pferd-${{ matrix.os }}
|
||||
name: Binaries
|
||||
path: dist/pferd-${{ matrix.os }}
|
||||
|
||||
release:
|
||||
@ -61,20 +57,18 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Download binaries
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
pattern: pferd-*
|
||||
merge-multiple: true
|
||||
name: Binaries
|
||||
|
||||
- name: Rename binaries
|
||||
run: |
|
||||
mv pferd-ubuntu-latest pferd-linux
|
||||
mv pferd-windows-latest pferd-windows.exe
|
||||
mv pferd-macos-13 pferd-mac-x86_64
|
||||
mv pferd-macos-latest pferd-mac
|
||||
|
||||
- name: Create release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
@ -82,4 +76,3 @@ jobs:
|
||||
pferd-linux
|
||||
pferd-windows.exe
|
||||
pferd-mac
|
||||
pferd-mac-x86_64
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,8 +2,6 @@
|
||||
/.venv/
|
||||
/PFERD.egg-info/
|
||||
__pycache__/
|
||||
/.vscode/
|
||||
/.idea/
|
||||
|
||||
# pyinstaller
|
||||
/pferd.spec
|
||||
|
||||
8
.vscode/settings.json
vendored
Normal file
8
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
{
|
||||
"files.insertFinalNewline": true,
|
||||
"files.trimFinalNewlines": true,
|
||||
"python.formatting.provider": "autopep8",
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.mypyEnabled": true,
|
||||
}
|
||||
66
CHANGELOG.md
66
CHANGELOG.md
@ -22,76 +22,10 @@ ambiguous situations.
|
||||
|
||||
## Unreleased
|
||||
|
||||
## 3.7.0 - 2024-11-13
|
||||
|
||||
### Added
|
||||
- Support for MOB videos in page descriptions
|
||||
- Clickable links in the report to directly open new/modified/not-deleted files
|
||||
- Support for non KIT shibboleth login
|
||||
|
||||
### Changed
|
||||
- Remove videos from description pages
|
||||
- Perform ILIAS cycle detection after processing the transform to allow
|
||||
ignoring duplicated elements
|
||||
- Parse headings (h1-h3) as folders in kit-ipd crawler
|
||||
|
||||
### Fixed
|
||||
- Personal desktop/dashboard/favorites crawling
|
||||
- Crawling of nested courses
|
||||
- Downloading of links with no target URL
|
||||
- Handle row flex on description pages
|
||||
- Add `<!DOCTYPE html>` heading to forum threads to fix mime type detection
|
||||
- Handle groups in cards
|
||||
|
||||
## 3.6.0 - 2024-10-23
|
||||
|
||||
### Added
|
||||
- Generic `ilias-web` crawler and `ilias-web` CLI command
|
||||
- Support for the course overview page. Using this URL as a target might cause
|
||||
duplication warnings, as subgroups are listed separately.
|
||||
- Support for named capture groups in regex transforms
|
||||
- Crawl custom item groups as folders
|
||||
|
||||
### Fixed
|
||||
- Normalization of meeting names in cards
|
||||
- Sanitization of slashes in exercise container names
|
||||
|
||||
## 3.5.2 - 2024-04-14
|
||||
|
||||
### Fixed
|
||||
- Crawling of personal desktop with ILIAS 8
|
||||
- Crawling of empty personal desktops
|
||||
|
||||
## 3.5.1 - 2024-04-09
|
||||
|
||||
### Added
|
||||
- Support for ILIAS 8
|
||||
|
||||
### Fixed
|
||||
- Video name deduplication
|
||||
|
||||
## 3.5.0 - 2023-09-13
|
||||
|
||||
### Added
|
||||
- `no-delete-prompt-override` conflict resolution strategy
|
||||
- Support for ILIAS learning modules
|
||||
- `show_not_deleted` option to stop printing the "Not Deleted" status or report
|
||||
message. This combines nicely with the `no-delete-prompt-override` strategy,
|
||||
causing PFERD to mostly ignore local-only files.
|
||||
- Support for mediacast video listings
|
||||
- Crawling of files in info tab
|
||||
|
||||
### Changed
|
||||
- Remove size suffix for files in content pages
|
||||
|
||||
### Fixed
|
||||
- Crawling of courses with the timeline view as the default tab
|
||||
- Crawling of file and custom opencast cards
|
||||
- Crawling of button cards without descriptions
|
||||
- Abort crawling when encountering an unexpected ilias root page redirect
|
||||
- Sanitize ascii control characters on Windows
|
||||
- Crawling of paginated past meetings
|
||||
- Ignore SCORM learning modules
|
||||
|
||||
## 3.4.3 - 2022-11-29
|
||||
|
||||
|
||||
90
CONFIG.md
90
CONFIG.md
@ -4,11 +4,11 @@ A config file consists of sections. A section begins with a `[section]` header,
|
||||
which is followed by a list of `key = value` pairs. Comments must be on their
|
||||
own line and start with `#`. Multiline values must be indented beyond their key.
|
||||
Boolean values can be `yes` or `no`. For more details and some examples on the
|
||||
format, see the [configparser documentation][cp-file]
|
||||
([interpolation][cp-interp] is disabled).
|
||||
format, see the [configparser documentation][1] ([interpolation][2] is
|
||||
disabled).
|
||||
|
||||
[cp-file]: <https://docs.python.org/3/library/configparser.html#supported-ini-file-structure> "Supported INI File Structure"
|
||||
[cp-interp]: <https://docs.python.org/3/library/configparser.html#interpolation-of-values> "Interpolation of values"
|
||||
[1]: <https://docs.python.org/3/library/configparser.html#supported-ini-file-structure> "Supported INI File Structure"
|
||||
[2]: <https://docs.python.org/3/library/configparser.html#interpolation-of-values> "Interpolation of values"
|
||||
|
||||
## The `DEFAULT` section
|
||||
|
||||
@ -26,9 +26,6 @@ default values for the other sections.
|
||||
`Added ...`) while running a crawler. (Default: `yes`)
|
||||
- `report`: Whether PFERD should print a report of added, changed and deleted
|
||||
local files for all crawlers before exiting. (Default: `yes`)
|
||||
- `show_not_deleted`: Whether PFERD should print messages in status and report
|
||||
when a local-only file wasn't deleted. Combines nicely with the
|
||||
`no-delete-prompt-override` conflict resolution strategy.
|
||||
- `share_cookies`: Whether crawlers should share cookies where applicable. For
|
||||
example, some crawlers share cookies if they crawl the same website using the
|
||||
same account. (Default: `yes`)
|
||||
@ -78,9 +75,6 @@ common to all crawlers:
|
||||
using `prompt` and always choosing "yes".
|
||||
- `no-delete`: Never delete local files, but overwrite local files if the
|
||||
remote file is different.
|
||||
- `no-delete-prompt-overwrite`: Never delete local files, but prompt to
|
||||
overwrite local files if the remote file is different. Combines nicely
|
||||
with the `show_not_deleted` option.
|
||||
- `transform`: Rules for renaming and excluding certain files and directories.
|
||||
For more details, see [this section](#transformation-rules). (Default: empty)
|
||||
- `tasks`: The maximum number of concurrent tasks (such as crawling or
|
||||
@ -146,7 +140,7 @@ crawler simulate a slower, network-based crawler.
|
||||
|
||||
This crawler crawls a KIT-IPD page by url. The root page can be crawled from
|
||||
outside the KIT network so you will be informed about any new/deleted files,
|
||||
but downloading files requires you to be within. Adding a short delay between
|
||||
but downloading files requires you to be within. Adding a show delay between
|
||||
requests is likely a good idea.
|
||||
|
||||
- `target`: URL to a KIT-IPD page
|
||||
@ -154,63 +148,6 @@ requests is likely a good idea.
|
||||
matches, the given link is downloaded as a file. This is used to extract
|
||||
files from KIT-IPD pages. (Default: `^.*?[^/]+\.(pdf|zip|c|cpp|java)$`)
|
||||
|
||||
### The `ilias-web` crawler
|
||||
|
||||
This crawler crawls a generic ILIAS instance.
|
||||
|
||||
Inspired by [this ILIAS downloader][ilias-dl], the following configurations should work
|
||||
out of the box for the corresponding universities:
|
||||
|
||||
[ilias-dl]: https://github.com/V3lop5/ilias-downloader/blob/main/configs "ilias-downloader configs"
|
||||
|
||||
| University | `base_url` | `login_type` | `client_id` |
|
||||
|---------------|-----------------------------------------|--------------|---------------|
|
||||
| FH Aachen | https://www.ili.fh-aachen.de | local | elearning |
|
||||
| Uni Köln | https://www.ilias.uni-koeln.de/ilias | local | uk |
|
||||
| Uni Konstanz | https://ilias.uni-konstanz.de | local | ILIASKONSTANZ |
|
||||
| Uni Stuttgart | https://ilias3.uni-stuttgart.de | local | Uni_Stuttgart |
|
||||
| Uni Tübingen | https://ovidius.uni-tuebingen.de/ilias3 | shibboleth | |
|
||||
|
||||
If your university isn't listed, try navigating to your instance's login page.
|
||||
Assuming no custom login service is used, the URL will look something like this:
|
||||
|
||||
```jinja
|
||||
{{ base_url }}/login.php?client_id={{ client_id }}&cmd=force_login&lang=
|
||||
```
|
||||
|
||||
If the values work, feel free to submit a PR and add them to the table above.
|
||||
|
||||
- `base_url`: The URL where the ILIAS instance is located. (Required)
|
||||
- `login_type`: How you authenticate. (Required)
|
||||
- `local`: Use `client_id` for authentication.
|
||||
- `shibboleth`: Use shibboleth for authentication.
|
||||
- `client_id`: An ID used for authentication if `login_type` is `local`. Is
|
||||
ignored if `login_type` is `shibboleth`.
|
||||
- `target`: The ILIAS element to crawl. (Required)
|
||||
- `desktop`: Crawl your personal desktop / dashboard
|
||||
- `<course id>`: Crawl the course with the given id
|
||||
- `<url>`: Crawl a given element by URL (preferably the permanent URL linked
|
||||
at the bottom of its ILIAS page).
|
||||
This also supports the "My Courses" overview page to download *all*
|
||||
courses. Note that this might produce confusing local directory layouts
|
||||
and duplication warnings if you are a member of an ILIAS group. The
|
||||
`desktop` target is generally preferable.
|
||||
- `auth`: Name of auth section to use for login. (Required)
|
||||
- `tfa_auth`: Name of auth section to use for two-factor authentication. Only
|
||||
uses the auth section's password. (Default: Anonymous `tfa` authenticator)
|
||||
- `links`: How to represent external links. (Default: `fancy`)
|
||||
- `ignore`: Don't download links.
|
||||
- `plaintext`: A text file containing only the URL.
|
||||
- `fancy`: A HTML file looking like the ILIAS link element.
|
||||
- `internet-shortcut`: An internet shortcut file (`.url` file).
|
||||
- `link_redirect_delay`: Time (in seconds) until `fancy` link files will
|
||||
redirect to the actual URL. Set to a negative value to disable the automatic
|
||||
redirect. (Default: `-1`)
|
||||
- `videos`: Whether to download videos. (Default: `no`)
|
||||
- `forums`: Whether to download forum threads. (Default: `no`)
|
||||
- `http_timeout`: The timeout (in seconds) for all HTTP requests. (Default:
|
||||
`20.0`)
|
||||
|
||||
### The `kit-ilias-web` crawler
|
||||
|
||||
This crawler crawls the KIT ILIAS instance.
|
||||
@ -289,10 +226,10 @@ is stored in the keyring.
|
||||
|
||||
### The `pass` authenticator
|
||||
|
||||
This authenticator queries the [`pass` password manager][pass] for a username
|
||||
and password. It tries to be mostly compatible with [browserpass][browserpass]
|
||||
and [passff][passff], so see those links for an overview of the format. If PFERD
|
||||
fails to load your password, you can use the `--explain` flag to see why.
|
||||
This authenticator queries the [`pass` password manager][3] for a username and
|
||||
password. It tries to be mostly compatible with [browserpass][4] and
|
||||
[passff][5], so see those links for an overview of the format. If PFERD fails
|
||||
to load your password, you can use the `--explain` flag to see why.
|
||||
|
||||
- `passname`: The name of the password to use (Required)
|
||||
- `username_prefixes`: A comma-separated list of username line prefixes
|
||||
@ -300,9 +237,9 @@ fails to load your password, you can use the `--explain` flag to see why.
|
||||
- `password_prefixes`: A comma-separated list of password line prefixes
|
||||
(Default: `password,pass,secret`)
|
||||
|
||||
[pass]: <https://www.passwordstore.org/> "Pass: The Standard Unix Password Manager"
|
||||
[browserpass]: <https://github.com/browserpass/browserpass-extension#organizing-password-store> "Organizing password store"
|
||||
[passff]: <https://github.com/passff/passff#multi-line-format> "Multi-line format"
|
||||
[3]: <https://www.passwordstore.org/> "Pass: The Standard Unix Password Manager"
|
||||
[4]: <https://github.com/browserpass/browserpass-extension#organizing-password-store> "Organizing password store"
|
||||
[5]: <https://github.com/passff/passff#multi-line-format> "Multi-line format"
|
||||
|
||||
### The `tfa` authenticator
|
||||
|
||||
@ -401,8 +338,7 @@ matches `SOURCE`, the output path is created using `TARGET` as template.
|
||||
be referred to as `{g<n>}` (e.g. `{g3}`). `{g0}` refers to the original path.
|
||||
If capturing group *n*'s contents are a valid integer, the integer value is
|
||||
available as `{i<n>}` (e.g. `{i3}`). If capturing group *n*'s contents are a
|
||||
valid float, the float value is available as `{f<n>}` (e.g. `{f3}`). Named capture
|
||||
groups (e.g. `(?P<name>)`) are available by their name (e.g. `{name}`). If a
|
||||
valid float, the float value is available as `{f<n>}` (e.g. `{f3}`). If a
|
||||
capturing group is not present (e.g. when matching the string `cd` with the
|
||||
regex `(ab)?cd`), the corresponding variables are not defined.
|
||||
|
||||
|
||||
5
LICENSE
5
LICENSE
@ -1,6 +1,5 @@
|
||||
Copyright 2019-2024 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
|
||||
TheChristophe, Scriptim, thelukasprobst, Toorero,
|
||||
Mr-Pine, p-fruck, PinieP
|
||||
Copyright 2019-2021 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
|
||||
TheChristophe, Scriptim, thelukasprobst, Toorero
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
|
||||
@ -47,8 +47,6 @@ def configure_logging_from_args(args: argparse.Namespace) -> None:
|
||||
log.output_explain = args.explain
|
||||
if args.status is not None:
|
||||
log.output_status = args.status
|
||||
if args.show_not_deleted is not None:
|
||||
log.output_not_deleted = args.show_not_deleted
|
||||
if args.report is not None:
|
||||
log.output_report = args.report
|
||||
|
||||
@ -74,8 +72,6 @@ def configure_logging_from_config(args: argparse.Namespace, config: Config) -> N
|
||||
log.output_status = config.default_section.status()
|
||||
if args.report is None:
|
||||
log.output_report = config.default_section.report()
|
||||
if args.show_not_deleted is None:
|
||||
log.output_not_deleted = config.default_section.show_not_deleted()
|
||||
except ConfigOptionError as e:
|
||||
log.error(str(e))
|
||||
sys.exit(1)
|
||||
|
||||
@ -8,7 +8,6 @@
|
||||
# well.
|
||||
|
||||
from . import command_local # noqa: F401 imported but unused
|
||||
from . import command_ilias_web # noqa: F401 imported but unused
|
||||
from . import command_kit_ilias_web # noqa: F401 imported but unused
|
||||
from . import command_kit_ipd # noqa: F401 imported but unused
|
||||
from .parser import PARSER, ParserLoadError, load_default_section # noqa: F401 imported but unused
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
import argparse
|
||||
import configparser
|
||||
|
||||
from ..logging import log
|
||||
from .common_ilias_args import configure_common_group_args, load_common
|
||||
from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
|
||||
|
||||
COMMAND_NAME = "ilias-web"
|
||||
|
||||
SUBPARSER = SUBPARSERS.add_parser(
|
||||
COMMAND_NAME,
|
||||
parents=[CRAWLER_PARSER],
|
||||
)
|
||||
|
||||
GROUP = SUBPARSER.add_argument_group(
|
||||
title=f"{COMMAND_NAME} crawler arguments",
|
||||
description=f"arguments for the '{COMMAND_NAME}' crawler",
|
||||
)
|
||||
|
||||
GROUP.add_argument(
|
||||
"--base-url",
|
||||
type=str,
|
||||
metavar="BASE_URL",
|
||||
help="The base url of the ilias instance"
|
||||
)
|
||||
|
||||
GROUP.add_argument(
|
||||
"--client-id",
|
||||
type=str,
|
||||
metavar="CLIENT_ID",
|
||||
help="The client id of the ilias instance"
|
||||
)
|
||||
|
||||
configure_common_group_args(GROUP)
|
||||
|
||||
|
||||
def load(
|
||||
args: argparse.Namespace,
|
||||
parser: configparser.ConfigParser,
|
||||
) -> None:
|
||||
log.explain(f"Creating config for command '{COMMAND_NAME}'")
|
||||
|
||||
parser["crawl:ilias"] = {}
|
||||
section = parser["crawl:ilias"]
|
||||
load_crawler(args, section)
|
||||
|
||||
section["type"] = COMMAND_NAME
|
||||
if args.ilias_url is not None:
|
||||
section["base_url"] = args.ilias_url
|
||||
if args.client_id is not None:
|
||||
section["client_id"] = args.client_id
|
||||
|
||||
load_common(section, args, parser)
|
||||
|
||||
|
||||
SUBPARSER.set_defaults(command=load)
|
||||
@ -1,37 +1,120 @@
|
||||
import argparse
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
|
||||
from ..crawl.ilias.file_templates import Links
|
||||
from ..logging import log
|
||||
from .common_ilias_args import configure_common_group_args, load_common
|
||||
from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
|
||||
|
||||
COMMAND_NAME = "kit-ilias-web"
|
||||
from .parser import (CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, ParserLoadError, load_crawler,
|
||||
show_value_error)
|
||||
|
||||
SUBPARSER = SUBPARSERS.add_parser(
|
||||
COMMAND_NAME,
|
||||
"kit-ilias-web",
|
||||
parents=[CRAWLER_PARSER],
|
||||
)
|
||||
|
||||
GROUP = SUBPARSER.add_argument_group(
|
||||
title=f"{COMMAND_NAME} crawler arguments",
|
||||
description=f"arguments for the '{COMMAND_NAME}' crawler",
|
||||
title="kit-ilias-web crawler arguments",
|
||||
description="arguments for the 'kit-ilias-web' crawler",
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"target",
|
||||
type=str,
|
||||
metavar="TARGET",
|
||||
help="course id, 'desktop', or ILIAS URL to crawl"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"output",
|
||||
type=Path,
|
||||
metavar="OUTPUT",
|
||||
help="output directory"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--username", "-u",
|
||||
type=str,
|
||||
metavar="USERNAME",
|
||||
help="user name for authentication"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--keyring",
|
||||
action=BooleanOptionalAction,
|
||||
help="use the system keyring to store and retrieve passwords"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--credential-file",
|
||||
type=Path,
|
||||
metavar="PATH",
|
||||
help="read username and password from a credential file"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--links",
|
||||
type=show_value_error(Links.from_string),
|
||||
metavar="OPTION",
|
||||
help="how to represent external links"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--link-redirect-delay",
|
||||
type=int,
|
||||
metavar="SECONDS",
|
||||
help="time before 'fancy' links redirect to to their target (-1 to disable)"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--videos",
|
||||
action=BooleanOptionalAction,
|
||||
help="crawl and download videos"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--forums",
|
||||
action=BooleanOptionalAction,
|
||||
help="crawl and download forum posts"
|
||||
)
|
||||
GROUP.add_argument(
|
||||
"--http-timeout", "-t",
|
||||
type=float,
|
||||
metavar="SECONDS",
|
||||
help="timeout for all HTTP requests"
|
||||
)
|
||||
|
||||
configure_common_group_args(GROUP)
|
||||
|
||||
|
||||
def load(
|
||||
args: argparse.Namespace,
|
||||
parser: configparser.ConfigParser,
|
||||
) -> None:
|
||||
log.explain(f"Creating config for command '{COMMAND_NAME}'")
|
||||
log.explain("Creating config for command 'kit-ilias-web'")
|
||||
|
||||
parser["crawl:ilias"] = {}
|
||||
section = parser["crawl:ilias"]
|
||||
load_crawler(args, section)
|
||||
|
||||
section["type"] = COMMAND_NAME
|
||||
load_common(section, args, parser)
|
||||
section["type"] = "kit-ilias-web"
|
||||
section["target"] = str(args.target)
|
||||
section["output_dir"] = str(args.output)
|
||||
section["auth"] = "auth:ilias"
|
||||
if args.links is not None:
|
||||
section["links"] = str(args.links.value)
|
||||
if args.link_redirect_delay is not None:
|
||||
section["link_redirect_delay"] = str(args.link_redirect_delay)
|
||||
if args.videos is not None:
|
||||
section["videos"] = "yes" if args.videos else "no"
|
||||
if args.forums is not None:
|
||||
section["forums"] = "yes" if args.forums else "no"
|
||||
if args.http_timeout is not None:
|
||||
section["http_timeout"] = str(args.http_timeout)
|
||||
|
||||
parser["auth:ilias"] = {}
|
||||
auth_section = parser["auth:ilias"]
|
||||
if args.credential_file is not None:
|
||||
if args.username is not None:
|
||||
raise ParserLoadError("--credential-file and --username can't be used together")
|
||||
if args.keyring:
|
||||
raise ParserLoadError("--credential-file and --keyring can't be used together")
|
||||
auth_section["type"] = "credential-file"
|
||||
auth_section["path"] = str(args.credential_file)
|
||||
elif args.keyring:
|
||||
auth_section["type"] = "keyring"
|
||||
else:
|
||||
auth_section["type"] = "simple"
|
||||
if args.username is not None:
|
||||
auth_section["username"] = args.username
|
||||
|
||||
|
||||
SUBPARSER.set_defaults(command=load)
|
||||
|
||||
@ -1,104 +0,0 @@
|
||||
import argparse
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
|
||||
from ..crawl.ilias.file_templates import Links
|
||||
from .parser import BooleanOptionalAction, ParserLoadError, show_value_error
|
||||
|
||||
|
||||
def configure_common_group_args(group: argparse._ArgumentGroup) -> None:
|
||||
"""These arguments are shared between the KIT and generic Ilias web command."""
|
||||
group.add_argument(
|
||||
"target",
|
||||
type=str,
|
||||
metavar="TARGET",
|
||||
help="course id, 'desktop', or ILIAS URL to crawl"
|
||||
)
|
||||
group.add_argument(
|
||||
"output",
|
||||
type=Path,
|
||||
metavar="OUTPUT",
|
||||
help="output directory"
|
||||
)
|
||||
group.add_argument(
|
||||
"--username", "-u",
|
||||
type=str,
|
||||
metavar="USERNAME",
|
||||
help="user name for authentication"
|
||||
)
|
||||
group.add_argument(
|
||||
"--keyring",
|
||||
action=BooleanOptionalAction,
|
||||
help="use the system keyring to store and retrieve passwords"
|
||||
)
|
||||
group.add_argument(
|
||||
"--credential-file",
|
||||
type=Path,
|
||||
metavar="PATH",
|
||||
help="read username and password from a credential file"
|
||||
)
|
||||
group.add_argument(
|
||||
"--links",
|
||||
type=show_value_error(Links.from_string),
|
||||
metavar="OPTION",
|
||||
help="how to represent external links"
|
||||
)
|
||||
group.add_argument(
|
||||
"--link-redirect-delay",
|
||||
type=int,
|
||||
metavar="SECONDS",
|
||||
help="time before 'fancy' links redirect to to their target (-1 to disable)"
|
||||
)
|
||||
group.add_argument(
|
||||
"--videos",
|
||||
action=BooleanOptionalAction,
|
||||
help="crawl and download videos"
|
||||
)
|
||||
group.add_argument(
|
||||
"--forums",
|
||||
action=BooleanOptionalAction,
|
||||
help="crawl and download forum posts"
|
||||
)
|
||||
group.add_argument(
|
||||
"--http-timeout", "-t",
|
||||
type=float,
|
||||
metavar="SECONDS",
|
||||
help="timeout for all HTTP requests"
|
||||
)
|
||||
|
||||
|
||||
def load_common(
|
||||
section: configparser.SectionProxy,
|
||||
args: argparse.Namespace,
|
||||
parser: configparser.ConfigParser,
|
||||
) -> None:
|
||||
"""Load common config between generic and KIT ilias web command"""
|
||||
section["target"] = str(args.target)
|
||||
section["output_dir"] = str(args.output)
|
||||
section["auth"] = "auth:ilias"
|
||||
if args.links is not None:
|
||||
section["links"] = str(args.links.value)
|
||||
if args.link_redirect_delay is not None:
|
||||
section["link_redirect_delay"] = str(args.link_redirect_delay)
|
||||
if args.videos is not None:
|
||||
section["videos"] = "yes" if args.videos else "no"
|
||||
if args.forums is not None:
|
||||
section["forums"] = "yes" if args.forums else "no"
|
||||
if args.http_timeout is not None:
|
||||
section["http_timeout"] = str(args.http_timeout)
|
||||
|
||||
parser["auth:ilias"] = {}
|
||||
auth_section = parser["auth:ilias"]
|
||||
if args.credential_file is not None:
|
||||
if args.username is not None:
|
||||
raise ParserLoadError("--credential-file and --username can't be used together")
|
||||
if args.keyring:
|
||||
raise ParserLoadError("--credential-file and --keyring can't be used together")
|
||||
auth_section["type"] = "credential-file"
|
||||
auth_section["path"] = str(args.credential_file)
|
||||
elif args.keyring:
|
||||
auth_section["type"] = "keyring"
|
||||
else:
|
||||
auth_section["type"] = "simple"
|
||||
if args.username is not None:
|
||||
auth_section["username"] = args.username
|
||||
@ -215,11 +215,6 @@ PARSER.add_argument(
|
||||
action=BooleanOptionalAction,
|
||||
help="whether crawlers should share cookies where applicable"
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--show-not-deleted",
|
||||
action=BooleanOptionalAction,
|
||||
help="print messages in status and report when PFERD did not delete a local only file"
|
||||
)
|
||||
|
||||
|
||||
def load_default_section(
|
||||
@ -238,8 +233,6 @@ def load_default_section(
|
||||
section["report"] = "yes" if args.report else "no"
|
||||
if args.share_cookies is not None:
|
||||
section["share_cookies"] = "yes" if args.share_cookies else "no"
|
||||
if args.show_not_deleted is not None:
|
||||
section["show_not_deleted"] = "yes" if args.show_not_deleted else "no"
|
||||
|
||||
|
||||
SUBPARSERS = PARSER.add_subparsers(title="crawlers")
|
||||
|
||||
@ -82,9 +82,6 @@ class DefaultSection(Section):
|
||||
def report(self) -> bool:
|
||||
return self.s.getboolean("report", fallback=True)
|
||||
|
||||
def show_not_deleted(self) -> bool:
|
||||
return self.s.getboolean("show_not_deleted", fallback=True)
|
||||
|
||||
def share_cookies(self) -> bool:
|
||||
return self.s.getboolean("share_cookies", fallback=True)
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ from typing import Callable, Dict
|
||||
from ..auth import Authenticator
|
||||
from ..config import Config
|
||||
from .crawler import Crawler, CrawlError, CrawlerSection # noqa: F401
|
||||
from .ilias import IliasWebCrawler, IliasWebCrawlerSection, KitIliasWebCrawler, KitIliasWebCrawlerSection
|
||||
from .ilias import KitIliasWebCrawler, KitIliasWebCrawlerSection
|
||||
from .kit_ipd_crawler import KitIpdCrawler, KitIpdCrawlerSection
|
||||
from .local_crawler import LocalCrawler, LocalCrawlerSection
|
||||
|
||||
@ -18,8 +18,6 @@ CrawlerConstructor = Callable[[
|
||||
CRAWLERS: Dict[str, CrawlerConstructor] = {
|
||||
"local": lambda n, s, c, a:
|
||||
LocalCrawler(n, LocalCrawlerSection(s), c),
|
||||
"ilias-web": lambda n, s, c, a:
|
||||
IliasWebCrawler(n, IliasWebCrawlerSection(s), c, a),
|
||||
"kit-ilias-web": lambda n, s, c, a:
|
||||
KitIliasWebCrawler(n, KitIliasWebCrawlerSection(s), c, a),
|
||||
"kit-ipd": lambda n, s, c, a:
|
||||
|
||||
@ -9,7 +9,6 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Ty
|
||||
from ..auth import Authenticator
|
||||
from ..config import Config, Section
|
||||
from ..deduplicator import Deduplicator
|
||||
from ..limiter import Limiter
|
||||
from ..logging import ProgressBar, log
|
||||
from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload
|
||||
from ..report import MarkConflictError, MarkDuplicateError, Report
|
||||
@ -98,10 +97,9 @@ def anoncritical(f: AWrapped) -> AWrapped:
|
||||
|
||||
|
||||
class CrawlToken(ReusableAsyncContextManager[ProgressBar]):
|
||||
def __init__(self, limiter: Limiter, path: PurePath):
|
||||
def __init__(self, path: PurePath):
|
||||
super().__init__()
|
||||
|
||||
self._limiter = limiter
|
||||
self._path = path
|
||||
|
||||
@property
|
||||
@ -110,17 +108,15 @@ class CrawlToken(ReusableAsyncContextManager[ProgressBar]):
|
||||
|
||||
async def _on_aenter(self) -> ProgressBar:
|
||||
self._stack.callback(lambda: log.status("[bold cyan]", "Crawled", fmt_path(self._path)))
|
||||
await self._stack.enter_async_context(self._limiter.limit_crawl())
|
||||
bar = self._stack.enter_context(log.crawl_bar("[bold bright_cyan]", "Crawling", fmt_path(self._path)))
|
||||
|
||||
return bar
|
||||
|
||||
|
||||
class DownloadToken(ReusableAsyncContextManager[Tuple[ProgressBar, FileSink]]):
|
||||
def __init__(self, limiter: Limiter, fs_token: FileSinkToken, path: PurePath):
|
||||
def __init__(self, fs_token: FileSinkToken, path: PurePath):
|
||||
super().__init__()
|
||||
|
||||
self._limiter = limiter
|
||||
self._fs_token = fs_token
|
||||
self._path = path
|
||||
|
||||
@ -129,7 +125,6 @@ class DownloadToken(ReusableAsyncContextManager[Tuple[ProgressBar, FileSink]]):
|
||||
return self._path
|
||||
|
||||
async def _on_aenter(self) -> Tuple[ProgressBar, FileSink]:
|
||||
await self._stack.enter_async_context(self._limiter.limit_download())
|
||||
sink = await self._stack.enter_async_context(self._fs_token)
|
||||
# The "Downloaded ..." message is printed in the output dir, not here
|
||||
bar = self._stack.enter_context(log.download_bar("[bold bright_cyan]", "Downloading",
|
||||
@ -235,12 +230,6 @@ class Crawler(ABC):
|
||||
self.name = name
|
||||
self.error_free = True
|
||||
|
||||
self._limiter = Limiter(
|
||||
task_limit=section.tasks(),
|
||||
download_limit=section.downloads(),
|
||||
task_delay=section.task_delay(),
|
||||
)
|
||||
|
||||
self._deduplicator = Deduplicator(section.windows_paths())
|
||||
self._transformer = Transformer(section.transform())
|
||||
|
||||
@ -258,10 +247,6 @@ class Crawler(ABC):
|
||||
def prev_report(self) -> Optional[Report]:
|
||||
return self._output_dir.prev_report
|
||||
|
||||
@property
|
||||
def output_dir(self) -> OutputDirectory:
|
||||
return self._output_dir
|
||||
|
||||
@staticmethod
|
||||
async def gather(awaitables: Sequence[Awaitable[Any]]) -> List[Any]:
|
||||
"""
|
||||
@ -292,13 +277,11 @@ class Crawler(ABC):
|
||||
return None
|
||||
|
||||
log.explain("Answer: Yes")
|
||||
return CrawlToken(self._limiter, path)
|
||||
return CrawlToken(path)
|
||||
|
||||
async def download(
|
||||
self,
|
||||
path: PurePath,
|
||||
*,
|
||||
etag_differs: Optional[bool] = None,
|
||||
mtime: Optional[datetime] = None,
|
||||
redownload: Optional[Redownload] = None,
|
||||
on_conflict: Optional[OnConflict] = None,
|
||||
@ -313,20 +296,13 @@ class Crawler(ABC):
|
||||
log.status("[bold bright_black]", "Ignored", fmt_path(path))
|
||||
return None
|
||||
|
||||
fs_token = await self._output_dir.download(
|
||||
path,
|
||||
transformed_path,
|
||||
etag_differs=etag_differs,
|
||||
mtime=mtime,
|
||||
redownload=redownload,
|
||||
on_conflict=on_conflict
|
||||
)
|
||||
fs_token = await self._output_dir.download(path, transformed_path, mtime, redownload, on_conflict)
|
||||
if fs_token is None:
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
|
||||
log.explain("Answer: Yes")
|
||||
return DownloadToken(self._limiter, fs_token, path)
|
||||
return DownloadToken(fs_token, path)
|
||||
|
||||
async def _cleanup(self) -> None:
|
||||
log.explain_topic("Decision: Clean up files")
|
||||
|
||||
@ -1,14 +1,9 @@
|
||||
import asyncio
|
||||
import http.cookies
|
||||
import ssl
|
||||
from datetime import datetime
|
||||
from http.cookiejar import LWPCookieJar
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
import certifi
|
||||
from aiohttp.client import ClientTimeout
|
||||
from bs4 import Tag
|
||||
import requests
|
||||
|
||||
from ..auth import Authenticator
|
||||
from ..config import Config
|
||||
@ -17,8 +12,6 @@ from ..utils import fmt_real_path
|
||||
from ..version import NAME, VERSION
|
||||
from .crawler import Crawler, CrawlerSection
|
||||
|
||||
ETAGS_CUSTOM_REPORT_VALUE_KEY = "etags"
|
||||
|
||||
|
||||
class HttpCrawlerSection(CrawlerSection):
|
||||
def http_timeout(self) -> float:
|
||||
@ -39,9 +32,9 @@ class HttpCrawler(Crawler):
|
||||
|
||||
self._authentication_id = 0
|
||||
self._authentication_lock = asyncio.Lock()
|
||||
self._request_count = 0
|
||||
self._http_timeout = section.http_timeout()
|
||||
self._http_timeout = section.http_timeout() # TODO Use or remove
|
||||
|
||||
self._cookie_jar = LWPCookieJar()
|
||||
self._cookie_jar_path = self._output_dir.resolve(self.COOKIE_FILE)
|
||||
self._shared_cookie_jar_paths: Optional[List[Path]] = None
|
||||
self._shared_auth = shared_auth
|
||||
@ -61,7 +54,6 @@ class HttpCrawler(Crawler):
|
||||
# This should reduce the amount of requests we make: If an authentication is in progress
|
||||
# all future requests wait for authentication to complete.
|
||||
async with self._authentication_lock:
|
||||
self._request_count += 1
|
||||
return self._authentication_id
|
||||
|
||||
async def authenticate(self, caller_auth_id: int) -> None:
|
||||
@ -110,32 +102,13 @@ class HttpCrawler(Crawler):
|
||||
|
||||
self._shared_cookie_jar_paths.append(self._cookie_jar_path)
|
||||
|
||||
def _load_cookies_from_file(self, path: Path) -> None:
|
||||
jar: Any = http.cookies.SimpleCookie()
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for i, line in enumerate(f):
|
||||
# Names of headers are case insensitive
|
||||
if line[:11].lower() == "set-cookie:":
|
||||
jar.load(line[11:])
|
||||
else:
|
||||
log.explain(f"Line {i} doesn't start with 'Set-Cookie:', ignoring it")
|
||||
self._cookie_jar.update_cookies(jar)
|
||||
|
||||
def _save_cookies_to_file(self, path: Path) -> None:
|
||||
jar: Any = http.cookies.SimpleCookie()
|
||||
for morsel in self._cookie_jar:
|
||||
jar[morsel.key] = morsel
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(jar.output(sep="\n"))
|
||||
f.write("\n") # A trailing newline is just common courtesy
|
||||
|
||||
def _load_cookies(self) -> None:
|
||||
log.explain_topic("Loading cookies")
|
||||
|
||||
cookie_jar_path: Optional[Path] = None
|
||||
|
||||
if self._shared_cookie_jar_paths is None:
|
||||
log.explain("Not sharing any cookies")
|
||||
log.explain("Not sharing cookies")
|
||||
cookie_jar_path = self._cookie_jar_path
|
||||
else:
|
||||
log.explain("Sharing cookies")
|
||||
@ -158,123 +131,38 @@ class HttpCrawler(Crawler):
|
||||
|
||||
log.explain(f"Loading cookies from {fmt_real_path(cookie_jar_path)}")
|
||||
try:
|
||||
self._load_cookies_from_file(cookie_jar_path)
|
||||
self._cookie_jar.load(filename=str(cookie_jar_path))
|
||||
except Exception as e:
|
||||
log.explain("Failed to load cookies")
|
||||
log.explain(str(e))
|
||||
log.explain(f"Failed to load cookies: {e}")
|
||||
log.explain("Proceeding without cookies")
|
||||
|
||||
def _save_cookies(self) -> None:
|
||||
log.explain_topic("Saving cookies")
|
||||
|
||||
try:
|
||||
log.explain(f"Saving cookies to {fmt_real_path(self._cookie_jar_path)}")
|
||||
self._save_cookies_to_file(self._cookie_jar_path)
|
||||
self._cookie_jar.save(filename=str(self._cookie_jar_path))
|
||||
except Exception as e:
|
||||
log.warn(f"Failed to save cookies to {fmt_real_path(self._cookie_jar_path)}")
|
||||
log.warn(str(e))
|
||||
|
||||
@staticmethod
|
||||
def get_folder_structure_from_heading_hierarchy(file_link: Tag, drop_h1: bool = False) -> PurePath:
|
||||
"""
|
||||
Retrieves the hierarchy of headings associated with the give file link and constructs a folder
|
||||
structure from them.
|
||||
|
||||
<h1> level headings usually only appear once and serve as the page title, so they would introduce
|
||||
redundant nesting. To avoid this, <h1> headings are ignored via the drop_h1 parameter.
|
||||
"""
|
||||
|
||||
def find_associated_headings(tag: Tag, level: int) -> PurePath:
|
||||
if level == 0 or (level == 1 and drop_h1):
|
||||
return PurePath()
|
||||
|
||||
level_heading = tag.find_previous(name=f"h{level}")
|
||||
|
||||
if level_heading is None:
|
||||
return find_associated_headings(tag, level - 1)
|
||||
|
||||
folder_name = level_heading.getText().strip()
|
||||
return find_associated_headings(level_heading, level - 1) / folder_name
|
||||
|
||||
# start at level <h3> because paragraph-level headings are usually too granular for folder names
|
||||
return find_associated_headings(file_link, 3)
|
||||
|
||||
def _get_previous_etag_from_report(self, path: PurePath) -> Optional[str]:
|
||||
"""
|
||||
If available, retrieves the entity tag for a given path which was stored in the previous report.
|
||||
"""
|
||||
if not self._output_dir.prev_report:
|
||||
return None
|
||||
|
||||
etags = self._output_dir.prev_report.get_custom_value(ETAGS_CUSTOM_REPORT_VALUE_KEY) or {}
|
||||
return etags.get(str(path))
|
||||
|
||||
def _add_etag_to_report(self, path: PurePath, etag: Optional[str]) -> None:
|
||||
"""
|
||||
Adds an entity tag for a given path to the report's custom values.
|
||||
"""
|
||||
if not etag:
|
||||
return
|
||||
|
||||
etags = self._output_dir.report.get_custom_value(ETAGS_CUSTOM_REPORT_VALUE_KEY) or {}
|
||||
etags[str(path)] = etag
|
||||
self._output_dir.report.add_custom_value(ETAGS_CUSTOM_REPORT_VALUE_KEY, etags)
|
||||
|
||||
async def _request_resource_version(self, resource_url: str) -> Tuple[Optional[str], Optional[datetime]]:
|
||||
"""
|
||||
Requests the ETag and Last-Modified headers of a resource via a HEAD request.
|
||||
If no entity tag / modification date can be obtained, the according value will be None.
|
||||
"""
|
||||
try:
|
||||
async with self.session.head(resource_url) as resp:
|
||||
if resp.status != 200:
|
||||
return None, None
|
||||
|
||||
etag_header = resp.headers.get("ETag")
|
||||
last_modified_header = resp.headers.get("Last-Modified")
|
||||
|
||||
if last_modified_header:
|
||||
try:
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified#directives
|
||||
datetime_format = "%a, %d %b %Y %H:%M:%S GMT"
|
||||
last_modified = datetime.strptime(last_modified_header, datetime_format)
|
||||
except ValueError:
|
||||
# last_modified remains None
|
||||
pass
|
||||
|
||||
return etag_header, last_modified
|
||||
except aiohttp.ClientError:
|
||||
return None, None
|
||||
log.warn(f"Failed to save cookies: {e}")
|
||||
|
||||
async def run(self) -> None:
|
||||
self._request_count = 0
|
||||
self._cookie_jar = aiohttp.CookieJar()
|
||||
self._load_cookies()
|
||||
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": f"{NAME}/{VERSION}"},
|
||||
cookie_jar=self._cookie_jar,
|
||||
connector=aiohttp.TCPConnector(ssl=ssl.create_default_context(cafile=certifi.where())),
|
||||
timeout=ClientTimeout(
|
||||
# 30 minutes. No download in the history of downloads was longer than 30 minutes.
|
||||
# This is enough to transfer a 600 MB file over a 3 Mib/s connection.
|
||||
# Allowing an arbitrary value could be annoying for overnight batch jobs
|
||||
total=15 * 60,
|
||||
connect=self._http_timeout,
|
||||
sock_connect=self._http_timeout,
|
||||
sock_read=self._http_timeout,
|
||||
),
|
||||
# See https://github.com/aio-libs/aiohttp/issues/6626
|
||||
# Without this aiohttp will mangle the redirect header from Shibboleth, invalidating the
|
||||
# passed signature. Shibboleth will not accept the broken signature and authentication will
|
||||
# fail.
|
||||
requote_redirect_url=False
|
||||
) as session:
|
||||
self.session = session
|
||||
self.session = requests.Session()
|
||||
self.session.headers["User-Agent"] = f"{NAME}/{VERSION}"
|
||||
|
||||
# From the request docs: "All requests code should work out of the box
|
||||
# with externally provided instances of CookieJar, e.g. LWPCookieJar and
|
||||
# FileCookieJar."
|
||||
# https://requests.readthedocs.io/en/latest/api/#requests.cookies.RequestsCookieJar
|
||||
self.session.cookies = self._cookie_jar # type: ignore
|
||||
|
||||
with self.session:
|
||||
try:
|
||||
await super().run()
|
||||
finally:
|
||||
del self.session
|
||||
log.explain_topic(f"Total amount of HTTP requests: {self._request_count}")
|
||||
|
||||
# They are saved in authenticate, but a final save won't hurt
|
||||
self._save_cookies()
|
||||
|
||||
@ -1,9 +1,3 @@
|
||||
from .kit_ilias_web_crawler import (IliasWebCrawler, IliasWebCrawlerSection, KitIliasWebCrawler,
|
||||
KitIliasWebCrawlerSection)
|
||||
from .kit_ilias_web_crawler import KitIliasWebCrawler, KitIliasWebCrawlerSection
|
||||
|
||||
__all__ = [
|
||||
"IliasWebCrawler",
|
||||
"IliasWebCrawlerSection",
|
||||
"KitIliasWebCrawler",
|
||||
"KitIliasWebCrawlerSection",
|
||||
]
|
||||
__all__ = ["KitIliasWebCrawler", "KitIliasWebCrawlerSection"]
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
import asyncio
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
from ...logging import log
|
||||
from ..crawler import AWrapped, CrawlError, CrawlWarning
|
||||
|
||||
|
||||
def _iorepeat(attempts: int, name: str, failure_is_error: bool = False) -> Callable[[AWrapped], AWrapped]:
|
||||
def decorator(f: AWrapped) -> AWrapped:
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
|
||||
last_exception: Optional[BaseException] = None
|
||||
for round in range(attempts):
|
||||
try:
|
||||
return await f(*args, **kwargs)
|
||||
except aiohttp.ContentTypeError: # invalid content type
|
||||
raise CrawlWarning("ILIAS returned an invalid content type")
|
||||
except aiohttp.TooManyRedirects:
|
||||
raise CrawlWarning("Got stuck in a redirect loop")
|
||||
except aiohttp.ClientPayloadError as e: # encoding or not enough bytes
|
||||
last_exception = e
|
||||
except aiohttp.ClientConnectionError as e: # e.g. timeout, disconnect, resolve failed, etc.
|
||||
last_exception = e
|
||||
except asyncio.exceptions.TimeoutError as e: # explicit http timeouts in HttpCrawler
|
||||
last_exception = e
|
||||
log.explain_topic(f"Retrying operation {name}. Retries left: {attempts - 1 - round}")
|
||||
log.explain(f"Last exception: {last_exception!r}")
|
||||
|
||||
if last_exception:
|
||||
message = f"Error in I/O Operation: {last_exception!r}"
|
||||
if failure_is_error:
|
||||
raise CrawlError(message) from last_exception
|
||||
else:
|
||||
raise CrawlWarning(message) from last_exception
|
||||
raise CrawlError("Impossible return in ilias _iorepeat")
|
||||
|
||||
return wrapper # type: ignore
|
||||
|
||||
return decorator
|
||||
@ -1,10 +1,6 @@
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
import bs4
|
||||
|
||||
from PFERD.utils import soupify
|
||||
|
||||
_link_template_plain = "{{link}}"
|
||||
_link_template_fancy = """
|
||||
<!DOCTYPE html>
|
||||
@ -98,71 +94,6 @@ _link_template_internet_shortcut = """
|
||||
URL={{link}}
|
||||
""".strip()
|
||||
|
||||
_learning_module_template = """
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>{{name}}</title>
|
||||
</head>
|
||||
|
||||
<style>
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.center-flex {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.nav {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
}
|
||||
</style>
|
||||
<body class="center-flex">
|
||||
{{body}}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def learning_module_template(body: bs4.Tag, name: str, prev: Optional[str], next: Optional[str]) -> str:
|
||||
# Seems to be comments, ignore those.
|
||||
for elem in body.select(".il-copg-mob-fullscreen-modal"):
|
||||
elem.decompose()
|
||||
|
||||
nav_template = """
|
||||
<div class="nav">
|
||||
{{left}}
|
||||
{{right}}
|
||||
</div>
|
||||
"""
|
||||
if prev and body.select_one(".ilc_page_lnav_LeftNavigation"):
|
||||
text = body.select_one(".ilc_page_lnav_LeftNavigation").getText().strip()
|
||||
left = f'<a href="{prev}">{text}</a>'
|
||||
else:
|
||||
left = "<span></span>"
|
||||
|
||||
if next and body.select_one(".ilc_page_rnav_RightNavigation"):
|
||||
text = body.select_one(".ilc_page_rnav_RightNavigation").getText().strip()
|
||||
right = f'<a href="{next}">{text}</a>'
|
||||
else:
|
||||
right = "<span></span>"
|
||||
|
||||
if top_nav := body.select_one(".ilc_page_tnav_TopNavigation"):
|
||||
top_nav.replace_with(
|
||||
soupify(nav_template.replace("{{left}}", left).replace("{{right}}", right).encode())
|
||||
)
|
||||
|
||||
if bot_nav := body.select_one(".ilc_page_bnav_BottomNavigation"):
|
||||
bot_nav.replace_with(soupify(nav_template.replace(
|
||||
"{{left}}", left).replace("{{right}}", right).encode())
|
||||
)
|
||||
|
||||
body = body.prettify()
|
||||
return _learning_module_template.replace("{{body}}", body).replace("{{name}}", name)
|
||||
|
||||
|
||||
class Links(Enum):
|
||||
IGNORE = "ignore"
|
||||
@ -171,24 +102,24 @@ class Links(Enum):
|
||||
INTERNET_SHORTCUT = "internet-shortcut"
|
||||
|
||||
def template(self) -> Optional[str]:
|
||||
if self == Links.FANCY:
|
||||
if self == self.FANCY:
|
||||
return _link_template_fancy
|
||||
elif self == Links.PLAINTEXT:
|
||||
elif self == self.PLAINTEXT:
|
||||
return _link_template_plain
|
||||
elif self == Links.INTERNET_SHORTCUT:
|
||||
elif self == self.INTERNET_SHORTCUT:
|
||||
return _link_template_internet_shortcut
|
||||
elif self == Links.IGNORE:
|
||||
elif self == self.IGNORE:
|
||||
return None
|
||||
raise ValueError("Missing switch case")
|
||||
|
||||
def extension(self) -> Optional[str]:
|
||||
if self == Links.FANCY:
|
||||
if self == self.FANCY:
|
||||
return ".html"
|
||||
elif self == Links.PLAINTEXT:
|
||||
elif self == self.PLAINTEXT:
|
||||
return ".txt"
|
||||
elif self == Links.INTERNET_SHORTCUT:
|
||||
elif self == self.INTERNET_SHORTCUT:
|
||||
return ".url"
|
||||
elif self == Links.IGNORE:
|
||||
elif self == self.IGNORE:
|
||||
return None
|
||||
raise ValueError("Missing switch case")
|
||||
|
||||
|
||||
@ -12,13 +12,6 @@ _STYLE_TAG_CONTENT = """
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.row-flex {
|
||||
display: flex;
|
||||
}
|
||||
.row-flex-wrap {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.accordion-head {
|
||||
background-color: #f5f7fa;
|
||||
padding: 0.5rem 0;
|
||||
@ -89,14 +82,9 @@ def clean(soup: BeautifulSoup) -> BeautifulSoup:
|
||||
dummy.decompose()
|
||||
if len(children) > 1:
|
||||
continue
|
||||
if isinstance(type(children[0]), Comment):
|
||||
if type(children[0]) == Comment:
|
||||
dummy.decompose()
|
||||
|
||||
# Delete video figures, as they can not be internalized anyway
|
||||
for video in soup.select(".ilc_media_cont_MediaContainerHighlighted .ilPageVideo"):
|
||||
if figure := video.find_parent("figure"):
|
||||
figure.decompose()
|
||||
|
||||
for hrule_imposter in soup.find_all(class_="ilc_section_Separator"):
|
||||
hrule_imposter.insert(0, soup.new_tag("hr"))
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -3,7 +3,7 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional, Union, cast
|
||||
from typing import Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
@ -15,27 +15,20 @@ TargetType = Union[str, int]
|
||||
|
||||
|
||||
class IliasElementType(Enum):
|
||||
BOOKING = "booking"
|
||||
COURSE = "course"
|
||||
EXERCISE = "exercise"
|
||||
EXERCISE_FILES = "exercise_files" # own submitted files
|
||||
TEST = "test" # an online test. Will be ignored currently.
|
||||
FILE = "file"
|
||||
FOLDER = "folder"
|
||||
FORUM = "forum"
|
||||
INFO_TAB = "info_tab"
|
||||
LEARNING_MODULE = "learning_module"
|
||||
LINK = "link"
|
||||
MEDIACAST_VIDEO = "mediacast_video"
|
||||
MEDIACAST_VIDEO_FOLDER = "mediacast_video_folder"
|
||||
BOOKING = "booking"
|
||||
MEETING = "meeting"
|
||||
MOB_VIDEO = "mob_video"
|
||||
OPENCAST_VIDEO = "opencast_video"
|
||||
OPENCAST_VIDEO_FOLDER = "opencast_video_folder"
|
||||
OPENCAST_VIDEO_FOLDER_MAYBE_PAGINATED = "opencast_video_folder_maybe_paginated"
|
||||
OPENCAST_VIDEO_PLAYER = "opencast_video_player"
|
||||
SCORM_LEARNING_MODULE = "scorm_learning_module"
|
||||
SURVEY = "survey"
|
||||
TEST = "test" # an online test. Will be ignored currently.
|
||||
VIDEO = "video"
|
||||
VIDEO_PLAYER = "video_player"
|
||||
VIDEO_FOLDER = "video_folder"
|
||||
VIDEO_FOLDER_MAYBE_PAGINATED = "video_folder_maybe_paginated"
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -50,13 +43,8 @@ class IliasPageElement:
|
||||
regexes = [
|
||||
r"eid=(?P<id>[0-9a-z\-]+)",
|
||||
r"file_(?P<id>\d+)",
|
||||
r"copa_(?P<id>\d+)",
|
||||
r"fold_(?P<id>\d+)",
|
||||
r"frm_(?P<id>\d+)",
|
||||
r"exc_(?P<id>\d+)",
|
||||
r"ref_id=(?P<id>\d+)",
|
||||
r"target=[a-z]+_(?P<id>\d+)",
|
||||
r"mm_(?P<id>\d+)"
|
||||
r"target=[a-z]+_(?P<id>\d+)"
|
||||
]
|
||||
|
||||
for regex in regexes:
|
||||
@ -67,52 +55,6 @@ class IliasPageElement:
|
||||
log.warn(f"Didn't find identity for {self.name} - {self.url}. Please report this.")
|
||||
return self.url
|
||||
|
||||
@staticmethod
|
||||
def create_new(
|
||||
typ: IliasElementType,
|
||||
url: str,
|
||||
name: str,
|
||||
mtime: Optional[datetime] = None,
|
||||
description: Optional[str] = None,
|
||||
skip_sanitize: bool = False
|
||||
) -> 'IliasPageElement':
|
||||
if typ == IliasElementType.MEETING:
|
||||
normalized = IliasPageElement._normalize_meeting_name(name)
|
||||
log.explain(f"Normalized meeting name from {name!r} to {normalized!r}")
|
||||
name = normalized
|
||||
|
||||
if not skip_sanitize:
|
||||
name = _sanitize_path_name(name)
|
||||
|
||||
return IliasPageElement(typ, url, name, mtime, description)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_meeting_name(meeting_name: str) -> str:
|
||||
"""
|
||||
Normalizes meeting names, which have a relative time as their first part,
|
||||
to their date in ISO format.
|
||||
"""
|
||||
|
||||
# This checks whether we can reach a `:` without passing a `-`
|
||||
if re.search(r"^[^-]+: ", meeting_name):
|
||||
# Meeting name only contains date: "05. Jan 2000:"
|
||||
split_delimiter = ":"
|
||||
else:
|
||||
# Meeting name contains date and start/end times: "05. Jan 2000, 16:00 - 17:30:"
|
||||
split_delimiter = ", "
|
||||
|
||||
# We have a meeting day without time
|
||||
date_portion_str = meeting_name.split(split_delimiter)[0]
|
||||
date_portion = demangle_date(date_portion_str)
|
||||
|
||||
# We failed to parse the date, bail out
|
||||
if not date_portion:
|
||||
return meeting_name
|
||||
|
||||
# Replace the first section with the absolute date
|
||||
rest_of_name = split_delimiter.join(meeting_name.split(split_delimiter)[1:])
|
||||
return datetime.strftime(date_portion, "%Y-%m-%d") + split_delimiter + rest_of_name
|
||||
|
||||
|
||||
@dataclass
|
||||
class IliasDownloadForumData:
|
||||
@ -129,14 +71,6 @@ class IliasForumThread:
|
||||
mtime: Optional[datetime]
|
||||
|
||||
|
||||
@dataclass
|
||||
class IliasLearningModulePage:
|
||||
title: str
|
||||
content: Tag
|
||||
next_url: Optional[str]
|
||||
previous_url: Optional[str]
|
||||
|
||||
|
||||
class IliasPage:
|
||||
|
||||
def __init__(self, soup: BeautifulSoup, _page_url: str, source_element: Optional[IliasPageElement]):
|
||||
@ -145,12 +79,6 @@ class IliasPage:
|
||||
self._page_type = source_element.type if source_element else None
|
||||
self._source_name = source_element.name if source_element else ""
|
||||
|
||||
@staticmethod
|
||||
def is_root_page(soup: BeautifulSoup) -> bool:
|
||||
if permalink := IliasPage.get_soup_permalink(soup):
|
||||
return "goto.php?target=root_" in permalink
|
||||
return False
|
||||
|
||||
def get_child_elements(self) -> List[IliasPageElement]:
|
||||
"""
|
||||
Return all child page elements you can find here.
|
||||
@ -158,9 +86,9 @@ class IliasPage:
|
||||
if self._is_video_player():
|
||||
log.explain("Page is a video player, extracting URL")
|
||||
return self._player_to_video()
|
||||
if self._is_opencast_video_listing():
|
||||
log.explain("Page is an opencast video listing, searching for elements")
|
||||
return self._find_opencast_video_entries()
|
||||
if self._is_video_listing():
|
||||
log.explain("Page is a video listing, searching for elements")
|
||||
return self._find_video_entries()
|
||||
if self._is_exercise_file():
|
||||
log.explain("Page is an exercise, searching for elements")
|
||||
return self._find_exercise_entries()
|
||||
@ -170,25 +98,9 @@ class IliasPage:
|
||||
if self._is_content_page():
|
||||
log.explain("Page is a content page, searching for elements")
|
||||
return self._find_copa_entries()
|
||||
if self._is_info_tab():
|
||||
log.explain("Page is info tab, searching for elements")
|
||||
return self._find_info_tab_entries()
|
||||
log.explain("Page is a normal folder, searching for elements")
|
||||
return self._find_normal_entries()
|
||||
|
||||
def get_info_tab(self) -> Optional[IliasPageElement]:
|
||||
tab: Optional[Tag] = self._soup.find(
|
||||
name="a",
|
||||
attrs={"href": lambda x: x and "cmdClass=ilinfoscreengui" in x}
|
||||
)
|
||||
if tab is not None:
|
||||
return IliasPageElement.create_new(
|
||||
IliasElementType.INFO_TAB,
|
||||
self._abs_url_from_link(tab),
|
||||
"infos"
|
||||
)
|
||||
return None
|
||||
|
||||
def get_description(self) -> Optional[BeautifulSoup]:
|
||||
def is_interesting_class(name: str) -> bool:
|
||||
return name in ["ilCOPageSection", "ilc_Paragraph", "ilc_va_ihcap_VAccordIHeadCap"]
|
||||
@ -214,34 +126,6 @@ class IliasPage:
|
||||
|
||||
return BeautifulSoup(raw_html, "html.parser")
|
||||
|
||||
def get_learning_module_data(self) -> Optional[IliasLearningModulePage]:
|
||||
if not self._is_learning_module_page():
|
||||
return None
|
||||
content = self._soup.select_one("#ilLMPageContent")
|
||||
title = self._soup.select_one(".ilc_page_title_PageTitle").getText().strip()
|
||||
return IliasLearningModulePage(
|
||||
title=title,
|
||||
content=content,
|
||||
next_url=self._find_learning_module_next(),
|
||||
previous_url=self._find_learning_module_prev()
|
||||
)
|
||||
|
||||
def _find_learning_module_next(self) -> Optional[str]:
|
||||
for link in self._soup.select("a.ilc_page_rnavlink_RightNavigationLink"):
|
||||
url = self._abs_url_from_link(link)
|
||||
if "baseClass=ilLMPresentationGUI" not in url:
|
||||
continue
|
||||
return url
|
||||
return None
|
||||
|
||||
def _find_learning_module_prev(self) -> Optional[str]:
|
||||
for link in self._soup.select("a.ilc_page_lnavlink_LeftNavigationLink"):
|
||||
url = self._abs_url_from_link(link)
|
||||
if "baseClass=ilLMPresentationGUI" not in url:
|
||||
continue
|
||||
return url
|
||||
return None
|
||||
|
||||
def get_download_forum_data(self) -> Optional[IliasDownloadForumData]:
|
||||
form = self._soup.find("form", attrs={"action": lambda x: x and "fallbackCmd=showThreads" in x})
|
||||
if not form:
|
||||
@ -268,18 +152,14 @@ class IliasPage:
|
||||
if self._is_ilias_opencast_embedding():
|
||||
log.explain("Unwrapping opencast embedding")
|
||||
return self.get_child_elements()[0]
|
||||
if self._page_type == IliasElementType.OPENCAST_VIDEO_FOLDER_MAYBE_PAGINATED:
|
||||
if self._page_type == IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED:
|
||||
log.explain("Unwrapping video pagination")
|
||||
return self._find_opencast_video_entries_paginated()[0]
|
||||
return self._find_video_entries_paginated()[0]
|
||||
if self._contains_collapsed_future_meetings():
|
||||
log.explain("Requesting *all* future meetings")
|
||||
return self._uncollapse_future_meetings_url()
|
||||
if not self._is_content_tab_selected():
|
||||
if self._page_type != IliasElementType.INFO_TAB:
|
||||
log.explain("Selecting content tab")
|
||||
return self._select_content_page_url()
|
||||
else:
|
||||
log.explain("Crawling info tab, skipping content select")
|
||||
return None
|
||||
|
||||
def _is_forum_page(self) -> bool:
|
||||
@ -292,7 +172,7 @@ class IliasPage:
|
||||
def _is_video_player(self) -> bool:
|
||||
return "paella_config_file" in str(self._soup)
|
||||
|
||||
def _is_opencast_video_listing(self) -> bool:
|
||||
def _is_video_listing(self) -> bool:
|
||||
if self._is_ilias_opencast_embedding():
|
||||
return True
|
||||
|
||||
@ -324,41 +204,27 @@ class IliasPage:
|
||||
return False
|
||||
|
||||
def _is_personal_desktop(self) -> bool:
|
||||
return "baseclass=ildashboardgui" in self._page_url.lower() and "&cmd=show" in self._page_url.lower()
|
||||
return self._soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x})
|
||||
|
||||
def _is_content_page(self) -> bool:
|
||||
if link := self.get_permalink():
|
||||
return "target=copa_" in link
|
||||
return False
|
||||
|
||||
def _is_learning_module_page(self) -> bool:
|
||||
if link := self.get_permalink():
|
||||
return "target=pg_" in link
|
||||
link = self._soup.find(id="current_perma_link")
|
||||
if not link:
|
||||
return False
|
||||
return "target=copa_" in link.get("value")
|
||||
|
||||
def _contains_collapsed_future_meetings(self) -> bool:
|
||||
return self._uncollapse_future_meetings_url() is not None
|
||||
|
||||
def _uncollapse_future_meetings_url(self) -> Optional[IliasPageElement]:
|
||||
element = self._soup.find(
|
||||
"a",
|
||||
attrs={"href": lambda x: x and ("crs_next_sess=1" in x or "crs_prev_sess=1" in x)}
|
||||
)
|
||||
element = self._soup.find("a", attrs={"href": lambda x: x and "crs_next_sess=1" in x})
|
||||
if not element:
|
||||
return None
|
||||
link = self._abs_url_from_link(element)
|
||||
return IliasPageElement.create_new(IliasElementType.FOLDER, link, "show all meetings")
|
||||
return IliasPageElement(IliasElementType.FOLDER, link, "show all meetings")
|
||||
|
||||
def _is_content_tab_selected(self) -> bool:
|
||||
return self._select_content_page_url() is None
|
||||
|
||||
def _is_info_tab(self) -> bool:
|
||||
might_be_info = self._soup.find("form", attrs={"name": lambda x: x == "formInfoScreen"}) is not None
|
||||
return self._page_type == IliasElementType.INFO_TAB and might_be_info
|
||||
|
||||
def _is_course_overview_page(self) -> bool:
|
||||
return "baseClass=ilmembershipoverviewgui" in self._page_url
|
||||
|
||||
def _select_content_page_url(self) -> Optional[IliasPageElement]:
|
||||
tab = self._soup.find(
|
||||
id="tab_view_content",
|
||||
@ -370,7 +236,7 @@ class IliasPage:
|
||||
link = tab.find("a")
|
||||
if link:
|
||||
link = self._abs_url_from_link(link)
|
||||
return IliasPageElement.create_new(IliasElementType.FOLDER, link, "select content page")
|
||||
return IliasPageElement(IliasElementType.FOLDER, link, "select content page")
|
||||
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not find content tab URL on {self._page_url!r}.")
|
||||
@ -400,16 +266,14 @@ class IliasPage:
|
||||
# and just fetch the lone video url!
|
||||
if len(streams) == 1:
|
||||
video_url = streams[0]["sources"]["mp4"][0]["src"]
|
||||
return [
|
||||
IliasPageElement.create_new(IliasElementType.OPENCAST_VIDEO, video_url, self._source_name)
|
||||
]
|
||||
return [IliasPageElement(IliasElementType.VIDEO, video_url, self._source_name)]
|
||||
|
||||
log.explain(f"Found multiple videos for stream at {self._source_name}")
|
||||
items = []
|
||||
for stream in sorted(streams, key=lambda stream: stream["content"]):
|
||||
full_name = f"{self._source_name.replace('.mp4', '')} ({stream['content']}).mp4"
|
||||
video_url = stream["sources"]["mp4"][0]["src"]
|
||||
items.append(IliasPageElement.create_new(IliasElementType.OPENCAST_VIDEO, video_url, full_name))
|
||||
items.append(IliasPageElement(IliasElementType.VIDEO, video_url, full_name))
|
||||
|
||||
return items
|
||||
|
||||
@ -424,26 +288,17 @@ class IliasPage:
|
||||
|
||||
link = self._abs_url_from_link(correct_link)
|
||||
|
||||
return IliasPageElement.create_new(IliasElementType.FORUM, link, "show all forum threads")
|
||||
return IliasPageElement(IliasElementType.FORUM, link, "show all forum threads")
|
||||
|
||||
def _find_personal_desktop_entries(self) -> List[IliasPageElement]:
|
||||
items: List[IliasPageElement] = []
|
||||
|
||||
titles: List[Tag] = self._soup.select("#block_pditems_0 .il-item-title")
|
||||
titles: List[Tag] = self._soup.select(".il-item-title")
|
||||
for title in titles:
|
||||
link = title.find("a")
|
||||
|
||||
if not link:
|
||||
log.explain(f"Skipping offline item: {title.getText().strip()!r}")
|
||||
continue
|
||||
|
||||
name = _sanitize_path_name(link.text.strip())
|
||||
url = self._abs_url_from_link(link)
|
||||
|
||||
if "cmd=manage" in url and "cmdClass=ilPDSelectedItemsBlockGUI" in url:
|
||||
# Configure button/link does not have anything interesting
|
||||
continue
|
||||
|
||||
type = self._find_type_from_link(name, link, url)
|
||||
if not type:
|
||||
_unexpected_html_warning()
|
||||
@ -456,7 +311,7 @@ class IliasPage:
|
||||
url = re.sub(r"(target=file_\d+)", r"\1_download", url)
|
||||
log.explain("Rewired file URL to include download part")
|
||||
|
||||
items.append(IliasPageElement.create_new(type, url, name))
|
||||
items.append(IliasPageElement(type, url, name))
|
||||
|
||||
return items
|
||||
|
||||
@ -466,36 +321,18 @@ class IliasPage:
|
||||
|
||||
for link in links:
|
||||
url = self._abs_url_from_link(link)
|
||||
name = re.sub(r"\([\d,.]+ [MK]B\)", "", link.getText()).strip().replace("\t", "")
|
||||
name = _sanitize_path_name(name)
|
||||
name = _sanitize_path_name(link.getText().strip().replace("\t", ""))
|
||||
|
||||
if "file_id" not in url:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Found unknown content page item {name!r} with url {url!r}")
|
||||
continue
|
||||
|
||||
items.append(IliasPageElement.create_new(IliasElementType.FILE, url, name))
|
||||
items.append(IliasPageElement(IliasElementType.FILE, url, name))
|
||||
|
||||
return items
|
||||
|
||||
def _find_info_tab_entries(self) -> List[IliasPageElement]:
|
||||
items = []
|
||||
links: List[Tag] = self._soup.select("a.il_ContainerItemCommand")
|
||||
|
||||
for link in links:
|
||||
if "cmdClass=ilobjcoursegui" not in link["href"]:
|
||||
continue
|
||||
if "cmd=sendfile" not in link["href"]:
|
||||
continue
|
||||
items.append(IliasPageElement.create_new(
|
||||
IliasElementType.FILE,
|
||||
self._abs_url_from_link(link),
|
||||
_sanitize_path_name(link.getText())
|
||||
))
|
||||
|
||||
return items
|
||||
|
||||
def _find_opencast_video_entries(self) -> List[IliasPageElement]:
|
||||
def _find_video_entries(self) -> List[IliasPageElement]:
|
||||
# ILIAS has three stages for video pages
|
||||
# 1. The initial dummy page without any videos. This page contains the link to the listing
|
||||
# 2. The video listing which might be paginated
|
||||
@ -515,29 +352,27 @@ class IliasPage:
|
||||
query_params = {"limit": "800", "cmd": "asyncGetTableGUI", "cmdMode": "asynch"}
|
||||
url = url_set_query_params(url, query_params)
|
||||
log.explain("Found ILIAS video frame page, fetching actual content next")
|
||||
return [
|
||||
IliasPageElement.create_new(IliasElementType.OPENCAST_VIDEO_FOLDER_MAYBE_PAGINATED, url, "")
|
||||
]
|
||||
return [IliasPageElement(IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED, url, "")]
|
||||
|
||||
is_paginated = self._soup.find(id=re.compile(r"tab_page_sel.+")) is not None
|
||||
|
||||
if is_paginated and not self._page_type == IliasElementType.OPENCAST_VIDEO_FOLDER:
|
||||
if is_paginated and not self._page_type == IliasElementType.VIDEO_FOLDER:
|
||||
# We are in stage 2 - try to break pagination
|
||||
return self._find_opencast_video_entries_paginated()
|
||||
return self._find_video_entries_paginated()
|
||||
|
||||
return self._find_opencast_video_entries_no_paging()
|
||||
return self._find_video_entries_no_paging()
|
||||
|
||||
def _find_opencast_video_entries_paginated(self) -> List[IliasPageElement]:
|
||||
def _find_video_entries_paginated(self) -> List[IliasPageElement]:
|
||||
table_element: Tag = self._soup.find(name="table", id=re.compile(r"tbl_xoct_.+"))
|
||||
|
||||
if table_element is None:
|
||||
log.warn("Couldn't increase elements per page (table not found). I might miss elements.")
|
||||
return self._find_opencast_video_entries_no_paging()
|
||||
return self._find_video_entries_no_paging()
|
||||
|
||||
id_match = re.match(r"tbl_xoct_(.+)", table_element.attrs["id"])
|
||||
if id_match is None:
|
||||
log.warn("Couldn't increase elements per page (table id not found). I might miss elements.")
|
||||
return self._find_opencast_video_entries_no_paging()
|
||||
return self._find_video_entries_no_paging()
|
||||
|
||||
table_id = id_match.group(1)
|
||||
|
||||
@ -546,9 +381,9 @@ class IliasPage:
|
||||
url = url_set_query_params(self._page_url, query_params)
|
||||
|
||||
log.explain("Disabled pagination, retrying folder as a new entry")
|
||||
return [IliasPageElement.create_new(IliasElementType.OPENCAST_VIDEO_FOLDER, url, "")]
|
||||
return [IliasPageElement(IliasElementType.VIDEO_FOLDER, url, "")]
|
||||
|
||||
def _find_opencast_video_entries_no_paging(self) -> List[IliasPageElement]:
|
||||
def _find_video_entries_no_paging(self) -> List[IliasPageElement]:
|
||||
"""
|
||||
Crawls the "second stage" video page. This page contains the actual video urls.
|
||||
"""
|
||||
@ -560,11 +395,11 @@ class IliasPage:
|
||||
results: List[IliasPageElement] = []
|
||||
|
||||
for link in video_links:
|
||||
results.append(self._listed_opencast_video_to_element(link))
|
||||
results.append(self._listed_video_to_element(link))
|
||||
|
||||
return results
|
||||
|
||||
def _listed_opencast_video_to_element(self, link: Tag) -> IliasPageElement:
|
||||
def _listed_video_to_element(self, link: Tag) -> IliasPageElement:
|
||||
# The link is part of a table with multiple columns, describing metadata.
|
||||
# 6th or 7th child (1 indexed) is the modification time string. Try to find it
|
||||
# by parsing backwards from the end and finding something that looks like a date
|
||||
@ -575,8 +410,8 @@ class IliasPage:
|
||||
modification_string = link.parent.parent.parent.select_one(
|
||||
f"td.std:nth-child({index})"
|
||||
).getText().strip()
|
||||
if match := re.search(r"\d+\.\d+.\d+ \d+:\d+", modification_string):
|
||||
modification_time = datetime.strptime(match.group(0), "%d.%m.%Y %H:%M")
|
||||
if re.search(r"\d+\.\d+.\d+ - \d+:\d+", modification_string):
|
||||
modification_time = datetime.strptime(modification_string, "%d.%m.%Y - %H:%M")
|
||||
break
|
||||
|
||||
if modification_time is None:
|
||||
@ -591,9 +426,7 @@ class IliasPage:
|
||||
video_url = self._abs_url_from_link(link)
|
||||
|
||||
log.explain(f"Found video {video_name!r} at {video_url}")
|
||||
return IliasPageElement.create_new(
|
||||
IliasElementType.OPENCAST_VIDEO_PLAYER, video_url, video_name, modification_time
|
||||
)
|
||||
return IliasPageElement(IliasElementType.VIDEO_PLAYER, video_url, video_name, modification_time)
|
||||
|
||||
def _find_exercise_entries(self) -> List[IliasPageElement]:
|
||||
if self._soup.find(id="tab_submission"):
|
||||
@ -627,7 +460,7 @@ class IliasPage:
|
||||
if date is None:
|
||||
log.warn(f"Date parsing failed for exercise entry {name!r}")
|
||||
|
||||
results.append(IliasPageElement.create_new(
|
||||
results.append(IliasPageElement(
|
||||
IliasElementType.FILE,
|
||||
self._abs_url_from_link(link),
|
||||
name,
|
||||
@ -660,22 +493,22 @@ class IliasPage:
|
||||
# Two divs, side by side. Left is the name, right is the link ==> get left
|
||||
# sibling
|
||||
file_name = file_link.parent.findPrevious(name="div").getText().strip()
|
||||
file_name = _sanitize_path_name(file_name)
|
||||
url = self._abs_url_from_link(file_link)
|
||||
|
||||
log.explain(f"Found exercise entry {file_name!r}")
|
||||
results.append(IliasPageElement.create_new(
|
||||
results.append(IliasPageElement(
|
||||
IliasElementType.FILE,
|
||||
url,
|
||||
_sanitize_path_name(container_name) + "/" + _sanitize_path_name(file_name),
|
||||
mtime=None, # We do not have any timestamp
|
||||
skip_sanitize=True
|
||||
container_name + "/" + file_name,
|
||||
None # We do not have any timestamp
|
||||
))
|
||||
|
||||
# Find all links to file listings (e.g. "Submitted Files" for groups)
|
||||
file_listings: List[Tag] = container.findAll(
|
||||
name="a",
|
||||
# download links contain the given command class
|
||||
attrs={"href": lambda x: x and "cmdclass=ilexsubmissionfilegui" in x.lower()}
|
||||
attrs={"href": lambda x: x and "cmdClass=ilexsubmissionfilegui" in x}
|
||||
)
|
||||
|
||||
# Add each listing as a new
|
||||
@ -686,15 +519,14 @@ class IliasPage:
|
||||
label_container: Tag = parent_container.find(
|
||||
attrs={"class": lambda x: x and "control-label" in x}
|
||||
)
|
||||
file_name = label_container.getText().strip()
|
||||
file_name = _sanitize_path_name(label_container.getText().strip())
|
||||
url = self._abs_url_from_link(listing)
|
||||
log.explain(f"Found exercise detail {file_name!r} at {url}")
|
||||
results.append(IliasPageElement.create_new(
|
||||
results.append(IliasPageElement(
|
||||
IliasElementType.EXERCISE_FILES,
|
||||
url,
|
||||
_sanitize_path_name(container_name) + "/" + _sanitize_path_name(file_name),
|
||||
None, # we do not have any timestamp
|
||||
skip_sanitize=True
|
||||
container_name + "/" + file_name,
|
||||
None # we do not have any timestamp
|
||||
))
|
||||
|
||||
return results
|
||||
@ -702,18 +534,12 @@ class IliasPage:
|
||||
def _find_normal_entries(self) -> List[IliasPageElement]:
|
||||
result: List[IliasPageElement] = []
|
||||
|
||||
links: List[Tag] = []
|
||||
# Fetch all links and throw them to the general interpreter
|
||||
if self._is_course_overview_page():
|
||||
log.explain("Page is a course overview page, adjusting link selector")
|
||||
links.extend(self._soup.select(".il-item-title > a"))
|
||||
else:
|
||||
links.extend(self._soup.select("a.il_ContainerItemTitle"))
|
||||
links: List[Tag] = self._soup.select("a.il_ContainerItemTitle")
|
||||
|
||||
for link in links:
|
||||
abs_url = self._abs_url_from_link(link)
|
||||
# Make sure parents are sanitized. We do not want accidental parents
|
||||
parents = [_sanitize_path_name(x) for x in self._find_upwards_folder_hierarchy(link)]
|
||||
parents = self._find_upwards_folder_hierarchy(link)
|
||||
|
||||
if parents:
|
||||
element_name = "/".join(parents) + "/" + _sanitize_path_name(link.getText())
|
||||
@ -731,94 +557,21 @@ class IliasPage:
|
||||
|
||||
if not element_type:
|
||||
continue
|
||||
if element_type == IliasElementType.MEETING:
|
||||
normalized = _sanitize_path_name(self._normalize_meeting_name(element_name))
|
||||
log.explain(f"Normalized meeting name from {element_name!r} to {normalized!r}")
|
||||
element_name = normalized
|
||||
elif element_type == IliasElementType.FILE:
|
||||
result.append(self._file_to_element(element_name, abs_url, link))
|
||||
continue
|
||||
|
||||
log.explain(f"Found {element_name!r}")
|
||||
result.append(IliasPageElement.create_new(
|
||||
element_type,
|
||||
abs_url,
|
||||
element_name,
|
||||
description=description,
|
||||
skip_sanitize=True
|
||||
))
|
||||
result.append(IliasPageElement(element_type, abs_url, element_name, description=description))
|
||||
|
||||
result += self._find_cards()
|
||||
result += self._find_mediacast_videos()
|
||||
result += self._find_mob_videos()
|
||||
|
||||
return result
|
||||
|
||||
def _find_mediacast_videos(self) -> List[IliasPageElement]:
|
||||
videos: List[IliasPageElement] = []
|
||||
|
||||
for elem in cast(List[Tag], self._soup.select(".ilPlayerPreviewOverlayOuter")):
|
||||
element_name = _sanitize_path_name(
|
||||
elem.select_one(".ilPlayerPreviewDescription").getText().strip()
|
||||
)
|
||||
if not element_name.endswith(".mp4"):
|
||||
# just to make sure it has some kinda-alrightish ending
|
||||
element_name = element_name + ".mp4"
|
||||
video_element = elem.find(name="video")
|
||||
if not video_element:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"No <video> element found for mediacast video '{element_name}'")
|
||||
continue
|
||||
|
||||
videos.append(IliasPageElement.create_new(
|
||||
typ=IliasElementType.MEDIACAST_VIDEO,
|
||||
url=self._abs_url_from_relative(video_element.get("src")),
|
||||
name=element_name,
|
||||
mtime=self._find_mediacast_video_mtime(elem.findParent(name="td"))
|
||||
))
|
||||
|
||||
return videos
|
||||
|
||||
def _find_mob_videos(self) -> List[IliasPageElement]:
|
||||
videos: List[IliasPageElement] = []
|
||||
|
||||
for figure in self._soup.select("figure.ilc_media_cont_MediaContainerHighlighted"):
|
||||
title = figure.select_one("figcaption").getText().strip() + ".mp4"
|
||||
video_element = figure.select_one("video")
|
||||
if not video_element:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"No <video> element found for mob video '{title}'")
|
||||
continue
|
||||
|
||||
url = None
|
||||
for source in video_element.select("source"):
|
||||
if source.get("type", "") == "video/mp4":
|
||||
url = source.get("src")
|
||||
break
|
||||
|
||||
if url is None:
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"No <source> element found for mob video '{title}'")
|
||||
continue
|
||||
|
||||
videos.append(IliasPageElement.create_new(
|
||||
typ=IliasElementType.MOB_VIDEO,
|
||||
url=self._abs_url_from_relative(url),
|
||||
name=_sanitize_path_name(title),
|
||||
mtime=None
|
||||
))
|
||||
|
||||
return videos
|
||||
|
||||
def _find_mediacast_video_mtime(self, enclosing_td: Tag) -> Optional[datetime]:
|
||||
description_td: Tag = enclosing_td.findPreviousSibling("td")
|
||||
if not description_td:
|
||||
return None
|
||||
|
||||
meta_tag: Tag = description_td.find_all("p")[-1]
|
||||
if not meta_tag:
|
||||
return None
|
||||
|
||||
updated_str = meta_tag.getText().strip().replace("\n", " ")
|
||||
updated_str = re.sub(".+?: ", "", updated_str)
|
||||
return demangle_date(updated_str)
|
||||
|
||||
def _is_in_expanded_meeting(self, tag: Tag) -> bool:
|
||||
"""
|
||||
Returns whether a file is part of an expanded meeting.
|
||||
@ -856,14 +609,11 @@ class IliasPage:
|
||||
# ILIAS has proper accordions and weird blocks that look like normal headings,
|
||||
# but some JS later transforms them into an accordion.
|
||||
|
||||
# This is for these weird JS-y blocks and custom item groups
|
||||
# This is for these weird JS-y blocks
|
||||
if "ilContainerItemsContainer" in parent.get("class"):
|
||||
data_store_url = parent.parent.get("data-store-url", "").lower()
|
||||
is_custom_item_group = "baseclass=ilcontainerblockpropertiesstoragegui" in data_store_url \
|
||||
and "cont_block_id=" in data_store_url
|
||||
# I am currently under the impression that *only* those JS blocks have an
|
||||
# ilNoDisplay class.
|
||||
if not is_custom_item_group and "ilNoDisplay" not in parent.get("class"):
|
||||
if "ilNoDisplay" not in parent.get("class"):
|
||||
continue
|
||||
prev: Tag = parent.findPreviousSibling("div")
|
||||
if "ilContainerBlockHeader" in prev.get("class"):
|
||||
@ -923,9 +673,7 @@ class IliasPage:
|
||||
full_path = name + "." + file_type
|
||||
|
||||
log.explain(f"Found file {full_path!r}")
|
||||
return IliasPageElement.create_new(
|
||||
IliasElementType.FILE, url, full_path, modification_date, skip_sanitize=True
|
||||
)
|
||||
return IliasPageElement(IliasElementType.FILE, url, full_path, modification_date)
|
||||
|
||||
def _find_cards(self) -> List[IliasPageElement]:
|
||||
result: List[IliasPageElement] = []
|
||||
@ -942,7 +690,7 @@ class IliasPage:
|
||||
log.warn_contd(f"Could not extract type for {title}")
|
||||
continue
|
||||
|
||||
result.append(IliasPageElement.create_new(type, url, name))
|
||||
result.append(IliasPageElement(type, url, name))
|
||||
|
||||
card_button_tiles: List[Tag] = self._soup.select(".card-title button")
|
||||
|
||||
@ -971,7 +719,7 @@ class IliasPage:
|
||||
log.warn_contd(f"Could not extract type for {button}")
|
||||
continue
|
||||
|
||||
result.append(IliasPageElement.create_new(type, url, name, description=description))
|
||||
result.append(IliasPageElement(type, url, name, description=description))
|
||||
|
||||
return result
|
||||
|
||||
@ -995,17 +743,13 @@ class IliasPage:
|
||||
icon: Tag = card_root.select_one(".il-card-repository-head .icon")
|
||||
|
||||
if "opencast" in icon["class"] or "xoct" in icon["class"]:
|
||||
return IliasElementType.OPENCAST_VIDEO_FOLDER_MAYBE_PAGINATED
|
||||
return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
|
||||
if "exc" in icon["class"]:
|
||||
return IliasElementType.EXERCISE
|
||||
if "grp" in icon["class"]:
|
||||
return IliasElementType.FOLDER
|
||||
if "webr" in icon["class"]:
|
||||
return IliasElementType.LINK
|
||||
if "book" in icon["class"]:
|
||||
return IliasElementType.BOOKING
|
||||
if "crsr" in icon["class"]:
|
||||
return IliasElementType.COURSE
|
||||
if "frm" in icon["class"]:
|
||||
return IliasElementType.FORUM
|
||||
if "sess" in icon["class"]:
|
||||
@ -1020,8 +764,6 @@ class IliasPage:
|
||||
return IliasElementType.SURVEY
|
||||
if "file" in icon["class"]:
|
||||
return IliasElementType.FILE
|
||||
if "mcst" in icon["class"]:
|
||||
return IliasElementType.MEDIACAST_VIDEO_FOLDER
|
||||
|
||||
_unexpected_html_warning()
|
||||
log.warn_contd(f"Could not extract type from {icon} for card title {card_title}")
|
||||
@ -1060,28 +802,6 @@ class IliasPage:
|
||||
if "cmdClass=ilobjtestgui" in parsed_url.query:
|
||||
return IliasElementType.TEST
|
||||
|
||||
if "baseClass=ilLMPresentationGUI" in parsed_url.query:
|
||||
return IliasElementType.LEARNING_MODULE
|
||||
|
||||
if "baseClass=ilMediaCastHandlerGUI" in parsed_url.query:
|
||||
return IliasElementType.MEDIACAST_VIDEO_FOLDER
|
||||
|
||||
if "baseClass=ilSAHSPresentationGUI" in parsed_url.query:
|
||||
return IliasElementType.SCORM_LEARNING_MODULE
|
||||
|
||||
# other universities might have content type specified in URL path
|
||||
if "_file_" in parsed_url.path:
|
||||
return IliasElementType.FILE
|
||||
|
||||
if "_fold_" in parsed_url.path or "_copa_" in parsed_url.path:
|
||||
return IliasElementType.FOLDER
|
||||
|
||||
if "_frm_" in parsed_url.path:
|
||||
return IliasElementType.FORUM
|
||||
|
||||
if "_exc_" in parsed_url.path:
|
||||
return IliasElementType.EXERCISE
|
||||
|
||||
# Booking and Meeting can not be detected based on the link. They do have a ref_id though, so
|
||||
# try to guess it from the image.
|
||||
|
||||
@ -1123,11 +843,7 @@ class IliasPage:
|
||||
if img_tag is None:
|
||||
img_tag = found_parent.select_one("img.icon")
|
||||
|
||||
is_session_expansion_button = found_parent.find(
|
||||
"a",
|
||||
attrs={"href": lambda x: x and ("crs_next_sess=" in x or "crs_prev_sess=" in x)}
|
||||
)
|
||||
if img_tag is None and is_session_expansion_button:
|
||||
if img_tag is None and found_parent.find("a", attrs={"href": lambda x: x and "crs_next_sess=" in x}):
|
||||
log.explain("Found session expansion button, skipping it as it has no content")
|
||||
return None
|
||||
|
||||
@ -1137,7 +853,7 @@ class IliasPage:
|
||||
return None
|
||||
|
||||
if "opencast" in str(img_tag["alt"]).lower():
|
||||
return IliasElementType.OPENCAST_VIDEO_FOLDER_MAYBE_PAGINATED
|
||||
return IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED
|
||||
|
||||
if str(img_tag["src"]).endswith("icon_exc.svg"):
|
||||
return IliasElementType.EXERCISE
|
||||
@ -1157,52 +873,34 @@ class IliasPage:
|
||||
if str(img_tag["src"]).endswith("icon_tst.svg"):
|
||||
return IliasElementType.TEST
|
||||
|
||||
if str(img_tag["src"]).endswith("icon_mcst.svg"):
|
||||
return IliasElementType.MEDIACAST_VIDEO_FOLDER
|
||||
|
||||
if str(img_tag["src"]).endswith("icon_sahs.svg"):
|
||||
return IliasElementType.SCORM_LEARNING_MODULE
|
||||
|
||||
return IliasElementType.FOLDER
|
||||
|
||||
@staticmethod
|
||||
def is_logged_in(soup: BeautifulSoup) -> bool:
|
||||
# Normal ILIAS pages
|
||||
mainbar: Optional[Tag] = soup.find(class_="il-maincontrols-metabar")
|
||||
if mainbar is not None:
|
||||
login_button = mainbar.find(attrs={"href": lambda x: x and "login.php" in x})
|
||||
shib_login = soup.find(id="button_shib_login")
|
||||
return not login_button and not shib_login
|
||||
def _normalize_meeting_name(meeting_name: str) -> str:
|
||||
"""
|
||||
Normalizes meeting names, which have a relative time as their first part,
|
||||
to their date in ISO format.
|
||||
"""
|
||||
|
||||
# Personal Desktop
|
||||
if soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x}):
|
||||
return True
|
||||
# This checks whether we can reach a `:` without passing a `-`
|
||||
if re.search(r"^[^-]+: ", meeting_name):
|
||||
# Meeting name only contains date: "05. Jan 2000:"
|
||||
split_delimiter = ":"
|
||||
else:
|
||||
# Meeting name contains date and start/end times: "05. Jan 2000, 16:00 - 17:30:"
|
||||
split_delimiter = ", "
|
||||
|
||||
# Empty personal desktop has zero (0) markers. Match on the text...
|
||||
if alert := soup.select_one(".alert-info"):
|
||||
text = alert.getText().lower()
|
||||
if "you have not yet selected any favourites" in text:
|
||||
return True
|
||||
if "sie haben aktuell noch keine favoriten ausgewählt" in text:
|
||||
return True
|
||||
# We have a meeting day without time
|
||||
date_portion_str = meeting_name.split(split_delimiter)[0]
|
||||
date_portion = demangle_date(date_portion_str)
|
||||
|
||||
# Video listing embeds do not have complete ILIAS html. Try to match them by
|
||||
# their video listing table
|
||||
video_table = soup.find(
|
||||
recursive=True,
|
||||
name="table",
|
||||
attrs={"id": lambda x: x is not None and x.startswith("tbl_xoct")}
|
||||
)
|
||||
if video_table is not None:
|
||||
return True
|
||||
# The individual video player wrapper page has nothing of the above.
|
||||
# Match it by its playerContainer.
|
||||
if soup.select_one("#playerContainer") is not None:
|
||||
return True
|
||||
return False
|
||||
# We failed to parse the date, bail out
|
||||
if not date_portion:
|
||||
return meeting_name
|
||||
|
||||
def get_permalink(self) -> Optional[str]:
|
||||
return IliasPage.get_soup_permalink(self._soup)
|
||||
# Replace the first section with the absolute date
|
||||
rest_of_name = split_delimiter.join(meeting_name.split(split_delimiter)[1:])
|
||||
return datetime.strftime(date_portion, "%Y-%m-%d") + split_delimiter + rest_of_name
|
||||
|
||||
def _abs_url_from_link(self, link_tag: Tag) -> str:
|
||||
"""
|
||||
@ -1216,13 +914,6 @@ class IliasPage:
|
||||
"""
|
||||
return urljoin(self._page_url, relative_url)
|
||||
|
||||
@staticmethod
|
||||
def get_soup_permalink(soup: BeautifulSoup) -> Optional[str]:
|
||||
perma_link_element: Tag = soup.select_one(".il-footer-permanent-url > a")
|
||||
if not perma_link_element or not perma_link_element.get("href"):
|
||||
return None
|
||||
return perma_link_element.get("href")
|
||||
|
||||
|
||||
def _unexpected_html_warning() -> None:
|
||||
log.warn("Encountered unexpected HTML structure, ignoring element.")
|
||||
|
||||
@ -1,37 +1,909 @@
|
||||
from typing import Dict, Literal
|
||||
import asyncio
|
||||
import re
|
||||
from collections.abc import Awaitable, Coroutine
|
||||
from pathlib import PurePath
|
||||
from typing import Any, Callable, Dict, List, Optional, Set, Union, cast
|
||||
|
||||
from ...auth import Authenticator
|
||||
import aiohttp
|
||||
import yarl
|
||||
from aiohttp import hdrs
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from ...auth import Authenticator, TfaAuthenticator
|
||||
from ...config import Config
|
||||
from .ilias_web_crawler import IliasWebCrawler, IliasWebCrawlerSection
|
||||
from .shibboleth_login import ShibbolethLogin
|
||||
from ...logging import ProgressBar, log
|
||||
from ...output_dir import FileSink, Redownload
|
||||
from ...utils import fmt_path, soupify, url_set_query_param
|
||||
from ..crawler import AWrapped, CrawlError, CrawlToken, CrawlWarning, DownloadToken, anoncritical
|
||||
from ..http_crawler import HttpCrawler, HttpCrawlerSection
|
||||
from .file_templates import Links
|
||||
from .ilias_html_cleaner import clean, insert_base_markup
|
||||
from .kit_ilias_html import (IliasElementType, IliasForumThread, IliasPage, IliasPageElement,
|
||||
_sanitize_path_name, parse_ilias_forum_export)
|
||||
|
||||
TargetType = Union[str, int]
|
||||
|
||||
_ILIAS_URL = "https://ilias.studium.kit.edu"
|
||||
|
||||
|
||||
class KitShibbolethBackgroundLoginSuccessful:
|
||||
class KitShibbolethBackgroundLoginSuccessful():
|
||||
pass
|
||||
|
||||
|
||||
class KitIliasWebCrawlerSection(IliasWebCrawlerSection):
|
||||
def base_url(self) -> str:
|
||||
return _ILIAS_URL
|
||||
class KitIliasWebCrawlerSection(HttpCrawlerSection):
|
||||
def target(self) -> TargetType:
|
||||
target = self.s.get("target")
|
||||
if not target:
|
||||
self.missing_value("target")
|
||||
|
||||
def login(self) -> Literal["shibboleth"]:
|
||||
return "shibboleth"
|
||||
if re.fullmatch(r"\d+", target):
|
||||
# Course id
|
||||
return int(target)
|
||||
if target == "desktop":
|
||||
# Full personal desktop
|
||||
return target
|
||||
if target.startswith(_ILIAS_URL):
|
||||
# ILIAS URL
|
||||
return target
|
||||
|
||||
self.invalid_value("target", target, "Should be <course id | desktop | kit ilias URL>")
|
||||
|
||||
def tfa_auth(self, authenticators: Dict[str, Authenticator]) -> Optional[Authenticator]:
|
||||
value: Optional[str] = self.s.get("tfa_auth")
|
||||
if value is None:
|
||||
return None
|
||||
auth = authenticators.get(value)
|
||||
if auth is None:
|
||||
self.invalid_value("tfa_auth", value, "No such auth section exists")
|
||||
return auth
|
||||
|
||||
def links(self) -> Links:
|
||||
type_str: Optional[str] = self.s.get("links")
|
||||
|
||||
if type_str is None:
|
||||
return Links.FANCY
|
||||
|
||||
try:
|
||||
return Links.from_string(type_str)
|
||||
except ValueError as e:
|
||||
self.invalid_value("links", type_str, str(e).capitalize())
|
||||
|
||||
def link_redirect_delay(self) -> int:
|
||||
return self.s.getint("link_redirect_delay", fallback=-1)
|
||||
|
||||
def videos(self) -> bool:
|
||||
return self.s.getboolean("videos", fallback=False)
|
||||
|
||||
def forums(self) -> bool:
|
||||
return self.s.getboolean("forums", fallback=False)
|
||||
|
||||
|
||||
class KitIliasWebCrawler(IliasWebCrawler):
|
||||
_DIRECTORY_PAGES: Set[IliasElementType] = set([
|
||||
IliasElementType.EXERCISE,
|
||||
IliasElementType.EXERCISE_FILES,
|
||||
IliasElementType.FOLDER,
|
||||
IliasElementType.MEETING,
|
||||
IliasElementType.VIDEO_FOLDER,
|
||||
IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED,
|
||||
])
|
||||
|
||||
_VIDEO_ELEMENTS: Set[IliasElementType] = set([
|
||||
IliasElementType.VIDEO,
|
||||
IliasElementType.VIDEO_PLAYER,
|
||||
IliasElementType.VIDEO_FOLDER,
|
||||
IliasElementType.VIDEO_FOLDER_MAYBE_PAGINATED,
|
||||
])
|
||||
|
||||
|
||||
def _iorepeat(attempts: int, name: str, failure_is_error: bool = False) -> Callable[[AWrapped], AWrapped]:
|
||||
def decorator(f: AWrapped) -> AWrapped:
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Optional[Any]:
|
||||
last_exception: Optional[BaseException] = None
|
||||
for round in range(attempts):
|
||||
try:
|
||||
return await f(*args, **kwargs)
|
||||
except aiohttp.ContentTypeError: # invalid content type
|
||||
raise CrawlWarning("ILIAS returned an invalid content type")
|
||||
except aiohttp.TooManyRedirects:
|
||||
raise CrawlWarning("Got stuck in a redirect loop")
|
||||
except aiohttp.ClientPayloadError as e: # encoding or not enough bytes
|
||||
last_exception = e
|
||||
except aiohttp.ClientConnectionError as e: # e.g. timeout, disconnect, resolve failed, etc.
|
||||
last_exception = e
|
||||
except asyncio.exceptions.TimeoutError as e: # explicit http timeouts in HttpCrawler
|
||||
last_exception = e
|
||||
log.explain_topic(f"Retrying operation {name}. Retries left: {attempts - 1 - round}")
|
||||
|
||||
if last_exception:
|
||||
message = f"Error in I/O Operation: {last_exception}"
|
||||
if failure_is_error:
|
||||
raise CrawlError(message) from last_exception
|
||||
else:
|
||||
raise CrawlWarning(message) from last_exception
|
||||
raise CrawlError("Impossible return in ilias _iorepeat")
|
||||
|
||||
return wrapper # type: ignore
|
||||
return decorator
|
||||
|
||||
|
||||
# Crawler control flow:
|
||||
#
|
||||
# crawl_desktop -+
|
||||
# |
|
||||
# crawl_course --+
|
||||
# |
|
||||
# @_io_repeat | # retries internally (before the bar)
|
||||
# +- crawl_url <-+
|
||||
# |
|
||||
# |
|
||||
# | @_wrap_io_exception # does not need to retry as children acquire bars
|
||||
# +> crawl_ilias_element -+
|
||||
# ^ |
|
||||
# | @_io_repeat | # retries internally (before the bar)
|
||||
# +- crawl_ilias_page <---+
|
||||
# | |
|
||||
# +> get_page | # Handles and retries authentication
|
||||
# |
|
||||
# @_io_repeat | # retries internally (before the bar)
|
||||
# +- download_link <---+
|
||||
# | |
|
||||
# +> resolve_target | # Handles and retries authentication
|
||||
# |
|
||||
# @_io_repeat | # retries internally (before the bar)
|
||||
# +- download_video <---+
|
||||
# | |
|
||||
# | @_io_repeat | # retries internally (before the bar)
|
||||
# +- download_file <---+
|
||||
# |
|
||||
# +> stream_from_url # Handles and retries authentication
|
||||
|
||||
class KitIliasWebCrawler(HttpCrawler):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
section: KitIliasWebCrawlerSection,
|
||||
config: Config,
|
||||
authenticators: Dict[str, Authenticator],
|
||||
authenticators: Dict[str, Authenticator]
|
||||
):
|
||||
super().__init__(name, section, config, authenticators)
|
||||
# Setting a main authenticator for cookie sharing
|
||||
auth = section.auth(authenticators)
|
||||
super().__init__(name, section, config, shared_auth=auth)
|
||||
|
||||
self._shibboleth_login = ShibbolethLogin(
|
||||
_ILIAS_URL,
|
||||
self._auth,
|
||||
if section.tasks() > 1:
|
||||
log.warn("""
|
||||
Please avoid using too many parallel requests as these are the KIT ILIAS
|
||||
instance's greatest bottleneck.
|
||||
""".strip())
|
||||
|
||||
self._shibboleth_login = KitShibbolethLogin(
|
||||
auth,
|
||||
section.tfa_auth(authenticators),
|
||||
)
|
||||
|
||||
self._base_url = _ILIAS_URL
|
||||
|
||||
self._target = section.target()
|
||||
self._link_file_redirect_delay = section.link_redirect_delay()
|
||||
self._links = section.links()
|
||||
self._videos = section.videos()
|
||||
self._forums = section.forums()
|
||||
self._visited_urls: Dict[str, PurePath] = dict()
|
||||
|
||||
async def _run(self) -> None:
|
||||
if isinstance(self._target, int):
|
||||
log.explain_topic(f"Inferred crawl target: Course with id {self._target}")
|
||||
await self._crawl_course(self._target)
|
||||
elif self._target == "desktop":
|
||||
log.explain_topic("Inferred crawl target: Personal desktop")
|
||||
await self._crawl_desktop()
|
||||
else:
|
||||
log.explain_topic(f"Inferred crawl target: URL {self._target}")
|
||||
await self._crawl_url(self._target)
|
||||
|
||||
async def _crawl_course(self, course_id: int) -> None:
|
||||
# Start crawling at the given course
|
||||
root_url = url_set_query_param(
|
||||
self._base_url + "/goto.php", "target", f"crs_{course_id}"
|
||||
)
|
||||
|
||||
await self._crawl_url(root_url, expected_id=course_id)
|
||||
|
||||
async def _crawl_desktop(self) -> None:
|
||||
appendix = r"ILIAS\PersonalDesktop\PDMainBarProvider|mm_pd_sel_items"
|
||||
appendix = appendix.encode("ASCII").hex()
|
||||
await self._crawl_url(self._base_url + "/gs_content.php?item=" + appendix)
|
||||
|
||||
async def _crawl_url(self, url: str, expected_id: Optional[int] = None) -> None:
|
||||
maybe_cl = await self.crawl(PurePath("."))
|
||||
if not maybe_cl:
|
||||
return
|
||||
cl = maybe_cl # Not mypy's fault, but explained here: https://github.com/python/mypy/issues/2608
|
||||
|
||||
def ensure_is_valid_course_id(parent: Optional[IliasPageElement], soup: BeautifulSoup) -> None:
|
||||
if parent is None and expected_id is not None:
|
||||
perma_link_element: Tag = soup.find(id="current_perma_link")
|
||||
if not perma_link_element or "crs_" not in perma_link_element.get("value"):
|
||||
raise CrawlError("Invalid course id? Didn't find anything looking like a course")
|
||||
|
||||
await self._crawl_ilias_page(url, None, cl, ensure_is_valid_course_id)
|
||||
|
||||
@anoncritical
|
||||
async def _crawl_ilias_page(
|
||||
self,
|
||||
url: str,
|
||||
parent: Optional[IliasPageElement],
|
||||
cl: CrawlToken,
|
||||
next_stage_hook: Callable[[Optional[IliasPageElement], BeautifulSoup], None] = lambda a, b: None
|
||||
) -> None:
|
||||
async with cl:
|
||||
next_stage_url: Optional[str] = url
|
||||
current_parent = parent
|
||||
|
||||
while next_stage_url:
|
||||
soup = await self._get_page(next_stage_url)
|
||||
log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
|
||||
log.explain(f"URL: {next_stage_url}")
|
||||
|
||||
next_stage_hook(current_parent, soup)
|
||||
|
||||
page = IliasPage(soup, next_stage_url, current_parent)
|
||||
if next_element := page.get_next_stage_element():
|
||||
current_parent = next_element
|
||||
next_stage_url = next_element.url
|
||||
else:
|
||||
next_stage_url = None
|
||||
|
||||
for element in sorted(page.get_child_elements(), key=lambda e: e.id()):
|
||||
await self._handle_ilias_element(cl.path, element)
|
||||
|
||||
if description_string := page.get_description():
|
||||
await self._download_description(cl.path, description_string)
|
||||
|
||||
# These decorators only apply *to this method* and *NOT* to the returned
|
||||
# awaitables!
|
||||
# This method does not await the handlers but returns them instead.
|
||||
# This ensures one level is handled at a time and name deduplication
|
||||
# works correctly.
|
||||
@anoncritical
|
||||
async def _handle_ilias_element(
|
||||
self,
|
||||
parent_path: PurePath,
|
||||
element: IliasPageElement,
|
||||
) -> None:
|
||||
if element.url in self._visited_urls:
|
||||
raise CrawlWarning(
|
||||
f"Found second path to element {element.name!r} at {element.url!r}. "
|
||||
+ f"First path: {fmt_path(self._visited_urls[element.url])}. "
|
||||
+ f"Second path: {fmt_path(parent_path)}."
|
||||
)
|
||||
self._visited_urls[element.url] = parent_path
|
||||
|
||||
element_path = PurePath(parent_path, element.name)
|
||||
|
||||
if element.type in _VIDEO_ELEMENTS:
|
||||
if not self._videos:
|
||||
log.status(
|
||||
"[bold bright_black]",
|
||||
"Ignored",
|
||||
fmt_path(element_path),
|
||||
"[bright_black](enable with option 'videos')"
|
||||
)
|
||||
return None
|
||||
|
||||
if element.type == IliasElementType.FILE:
|
||||
await self._handle_file(element, element_path)
|
||||
elif element.type == IliasElementType.FORUM:
|
||||
if not self._forums:
|
||||
log.status(
|
||||
"[bold bright_black]",
|
||||
"Ignored",
|
||||
fmt_path(element_path),
|
||||
"[bright_black](enable with option 'forums')"
|
||||
)
|
||||
return None
|
||||
await self._handle_forum(element, element_path)
|
||||
elif element.type == IliasElementType.TEST:
|
||||
log.status(
|
||||
"[bold bright_black]",
|
||||
"Ignored",
|
||||
fmt_path(element_path),
|
||||
"[bright_black](tests contain no relevant data)"
|
||||
)
|
||||
return None
|
||||
elif element.type == IliasElementType.SURVEY:
|
||||
log.status(
|
||||
"[bold bright_black]",
|
||||
"Ignored",
|
||||
fmt_path(element_path),
|
||||
"[bright_black](surveys contain no relevant data)"
|
||||
)
|
||||
return None
|
||||
elif element.type == IliasElementType.LINK:
|
||||
await self._handle_link(element, element_path)
|
||||
elif element.type == IliasElementType.BOOKING:
|
||||
await self._handle_booking(element, element_path)
|
||||
elif element.type == IliasElementType.VIDEO:
|
||||
await self._handle_file(element, element_path)
|
||||
elif element.type == IliasElementType.VIDEO_PLAYER:
|
||||
await self._handle_video(element, element_path)
|
||||
elif element.type in _DIRECTORY_PAGES:
|
||||
maybe_cl = await self.crawl(element_path)
|
||||
if not maybe_cl:
|
||||
return None
|
||||
await self._crawl_ilias_page(element.url, element, maybe_cl)
|
||||
else:
|
||||
# This will retry it a few times, failing everytime. It doesn't make any network
|
||||
# requests, so that's fine.
|
||||
raise CrawlWarning(f"Unknown element type: {element.type!r}")
|
||||
|
||||
async def _handle_link(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> None:
|
||||
log.explain_topic(f"Decision: Crawl Link {fmt_path(element_path)}")
|
||||
log.explain(f"Links type is {self._links}")
|
||||
|
||||
link_template_maybe = self._links.template()
|
||||
link_extension = self._links.extension()
|
||||
if not link_template_maybe or not link_extension:
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
else:
|
||||
log.explain("Answer: Yes")
|
||||
element_path = element_path.with_name(element_path.name + link_extension)
|
||||
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return None
|
||||
|
||||
await self._download_link(element, link_template_maybe, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "resolving link")
|
||||
async def _download_link(self, element: IliasPageElement, link_template: str, dl: DownloadToken) -> None:
|
||||
async with dl as (bar, sink):
|
||||
export_url = element.url.replace("cmd=calldirectlink", "cmd=exportHTML")
|
||||
real_url = await self._resolve_link_target(export_url)
|
||||
self._write_link_content(link_template, real_url, element.name, element.description, sink)
|
||||
|
||||
def _write_link_content(
|
||||
self,
|
||||
link_template: str,
|
||||
url: str,
|
||||
name: str,
|
||||
description: Optional[str],
|
||||
sink: FileSink,
|
||||
) -> None:
|
||||
content = link_template
|
||||
content = content.replace("{{link}}", url)
|
||||
content = content.replace("{{name}}", name)
|
||||
content = content.replace("{{description}}", str(description))
|
||||
content = content.replace("{{redirect_delay}}", str(self._link_file_redirect_delay))
|
||||
sink.file.write(content.encode("utf-8"))
|
||||
sink.done()
|
||||
|
||||
async def _handle_booking(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> Optional[Coroutine[Any, Any, None]]:
|
||||
log.explain_topic(f"Decision: Crawl Booking Link {fmt_path(element_path)}")
|
||||
log.explain(f"Links type is {self._links}")
|
||||
|
||||
link_template_maybe = self._links.template()
|
||||
link_extension = self._links.extension()
|
||||
if not link_template_maybe or not link_extension:
|
||||
log.explain("Answer: No")
|
||||
return None
|
||||
else:
|
||||
log.explain("Answer: Yes")
|
||||
element_path = element_path.with_name(element_path.name + link_extension)
|
||||
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return None
|
||||
|
||||
return self._download_booking(element, link_template_maybe, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(1, "downloading description")
|
||||
async def _download_description(self, parent_path: PurePath, description: BeautifulSoup) -> None:
|
||||
path = parent_path / "Description.html"
|
||||
dl = await self.download(path, redownload=Redownload.ALWAYS)
|
||||
if not dl:
|
||||
return
|
||||
|
||||
async with dl as (bar, sink):
|
||||
description = clean(insert_base_markup(description))
|
||||
sink.file.write(description.prettify().encode("utf-8"))
|
||||
sink.done()
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "resolving booking")
|
||||
async def _download_booking(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
link_template: str,
|
||||
dl: DownloadToken,
|
||||
) -> None:
|
||||
async with dl as (bar, sink):
|
||||
self._write_link_content(link_template, element.url, element.name, element.description, sink)
|
||||
|
||||
async def _resolve_link_target(self, export_url: str) -> str:
|
||||
async with self.session.get(export_url, allow_redirects=False) as resp:
|
||||
# No redirect means we were authenticated
|
||||
if hdrs.LOCATION not in resp.headers:
|
||||
return soupify(await resp.read()).select_one("a").get("href").strip()
|
||||
|
||||
await self._authenticate()
|
||||
|
||||
async with self.session.get(export_url, allow_redirects=False) as resp:
|
||||
# No redirect means we were authenticated
|
||||
if hdrs.LOCATION not in resp.headers:
|
||||
return soupify(await resp.read()).select_one("a").get("href").strip()
|
||||
|
||||
raise CrawlError("resolve_link_target failed even after authenticating")
|
||||
|
||||
async def _handle_video(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> None:
|
||||
# Copy old mapping as it is likely still relevant
|
||||
if self.prev_report:
|
||||
self.report.add_custom_value(
|
||||
str(element_path),
|
||||
self.prev_report.get_custom_value(str(element_path))
|
||||
)
|
||||
|
||||
# A video might contain other videos, so let's "crawl" the video first
|
||||
# to ensure rate limits apply. This must be a download as *this token*
|
||||
# is re-used if the video consists of a single stream. In that case the
|
||||
# file name is used and *not* the stream name the ilias html parser reported
|
||||
# to ensure backwards compatibility.
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime, redownload=Redownload.ALWAYS)
|
||||
|
||||
# If we do not want to crawl it (user filter) or we have every file
|
||||
# from the cached mapping already, we can ignore this and bail
|
||||
if not maybe_dl or self._all_videos_locally_present(element_path):
|
||||
# Mark all existing cideos as known so they do not get deleted
|
||||
# during dleanup. We "downloaded" them, just without actually making
|
||||
# a network request as we assumed they did not change.
|
||||
for video in self._previous_contained_videos(element_path):
|
||||
await self.download(video)
|
||||
|
||||
return None
|
||||
|
||||
await self._download_video(element_path, element, maybe_dl)
|
||||
|
||||
def _previous_contained_videos(self, video_path: PurePath) -> List[PurePath]:
|
||||
if not self.prev_report:
|
||||
return []
|
||||
custom_value = self.prev_report.get_custom_value(str(video_path))
|
||||
if not custom_value:
|
||||
return []
|
||||
names = cast(List[str], custom_value)
|
||||
folder = video_path.parent
|
||||
return [PurePath(folder, name) for name in names]
|
||||
|
||||
def _all_videos_locally_present(self, video_path: PurePath) -> bool:
|
||||
if contained_videos := self._previous_contained_videos(video_path):
|
||||
log.explain_topic(f"Checking local cache for video {video_path.name}")
|
||||
all_found_locally = True
|
||||
for video in contained_videos:
|
||||
transformed_path = self._to_local_video_path(video)
|
||||
if transformed_path:
|
||||
exists_locally = self._output_dir.resolve(transformed_path).exists()
|
||||
all_found_locally = all_found_locally and exists_locally
|
||||
if all_found_locally:
|
||||
log.explain("Found all videos locally, skipping enumeration request")
|
||||
return True
|
||||
log.explain("Missing at least one video, continuing with requests!")
|
||||
return False
|
||||
|
||||
def _to_local_video_path(self, path: PurePath) -> Optional[PurePath]:
|
||||
if transformed := self._transformer.transform(path):
|
||||
return self._deduplicator.fixup_path(transformed)
|
||||
return None
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "downloading video")
|
||||
async def _download_video(
|
||||
self,
|
||||
original_path: PurePath,
|
||||
element: IliasPageElement,
|
||||
dl: DownloadToken
|
||||
) -> None:
|
||||
stream_elements: List[IliasPageElement] = []
|
||||
async with dl as (bar, sink):
|
||||
page = IliasPage(await self._get_page(element.url), element.url, element)
|
||||
stream_elements = page.get_child_elements()
|
||||
|
||||
if len(stream_elements) > 1:
|
||||
log.explain(f"Found multiple video streams for {element.name}")
|
||||
else:
|
||||
log.explain(f"Using single video mode for {element.name}")
|
||||
stream_element = stream_elements[0]
|
||||
|
||||
transformed_path = self._to_local_video_path(original_path)
|
||||
if not transformed_path:
|
||||
raise CrawlError(f"Download returned a path but transform did not for {original_path}")
|
||||
|
||||
# We do not have a local cache yet
|
||||
if self._output_dir.resolve(transformed_path).exists():
|
||||
log.explain(f"Video for {element.name} existed locally")
|
||||
else:
|
||||
await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
|
||||
self.report.add_custom_value(str(original_path), [original_path.name])
|
||||
return
|
||||
|
||||
contained_video_paths: List[str] = []
|
||||
|
||||
for stream_element in stream_elements:
|
||||
video_path = original_path.parent / stream_element.name
|
||||
contained_video_paths.append(str(video_path))
|
||||
|
||||
maybe_dl = await self.download(video_path, mtime=element.mtime, redownload=Redownload.NEVER)
|
||||
if not maybe_dl:
|
||||
continue
|
||||
async with maybe_dl as (bar, sink):
|
||||
log.explain(f"Streaming video from real url {stream_element.url}")
|
||||
await self._stream_from_url(stream_element.url, sink, bar, is_video=True)
|
||||
|
||||
self.report.add_custom_value(str(original_path), contained_video_paths)
|
||||
|
||||
async def _handle_file(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> None:
|
||||
maybe_dl = await self.download(element_path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return None
|
||||
await self._download_file(element, maybe_dl)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "downloading file")
|
||||
async def _download_file(self, element: IliasPageElement, dl: DownloadToken) -> None:
|
||||
assert dl # The function is only reached when dl is not None
|
||||
async with dl as (bar, sink):
|
||||
await self._stream_from_url(element.url, sink, bar, is_video=False)
|
||||
|
||||
async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar, is_video: bool) -> None:
|
||||
async def try_stream() -> bool:
|
||||
async with self.session.get(url, allow_redirects=is_video) as resp:
|
||||
if not is_video:
|
||||
# Redirect means we weren't authenticated
|
||||
if hdrs.LOCATION in resp.headers:
|
||||
return False
|
||||
# we wanted a video but got HTML
|
||||
if is_video and "html" in resp.content_type:
|
||||
return False
|
||||
|
||||
if resp.content_length:
|
||||
bar.set_total(resp.content_length)
|
||||
|
||||
async for data in resp.content.iter_chunked(1024):
|
||||
sink.file.write(data)
|
||||
bar.advance(len(data))
|
||||
|
||||
sink.done()
|
||||
return True
|
||||
|
||||
auth_id = await self._current_auth_id()
|
||||
if await try_stream():
|
||||
return
|
||||
|
||||
await self.authenticate(auth_id)
|
||||
|
||||
if not await try_stream():
|
||||
raise CrawlError("File streaming failed after authenticate()")
|
||||
|
||||
async def _handle_forum(
|
||||
self,
|
||||
element: IliasPageElement,
|
||||
element_path: PurePath,
|
||||
) -> None:
|
||||
maybe_cl = await self.crawl(element_path)
|
||||
if not maybe_cl:
|
||||
return None
|
||||
await self._crawl_forum(element, maybe_cl)
|
||||
|
||||
@_iorepeat(3, "crawling forum")
|
||||
@anoncritical
|
||||
async def _crawl_forum(self, element: IliasPageElement, cl: CrawlToken) -> None:
|
||||
elements: List[IliasForumThread] = []
|
||||
|
||||
async with cl:
|
||||
next_stage_url = element.url
|
||||
while next_stage_url:
|
||||
log.explain_topic(f"Parsing HTML page for {fmt_path(cl.path)}")
|
||||
log.explain(f"URL: {next_stage_url}")
|
||||
|
||||
soup = await self._get_page(next_stage_url)
|
||||
page = IliasPage(soup, next_stage_url, None)
|
||||
|
||||
if next := page.get_next_stage_element():
|
||||
next_stage_url = next.url
|
||||
else:
|
||||
break
|
||||
|
||||
download_data = page.get_download_forum_data()
|
||||
if not download_data:
|
||||
raise CrawlWarning("Failed to extract forum data")
|
||||
if download_data.empty:
|
||||
log.explain("Forum had no threads")
|
||||
elements = []
|
||||
return
|
||||
html = await self._post_authenticated(download_data.url, download_data.form_data)
|
||||
elements = parse_ilias_forum_export(soupify(html))
|
||||
|
||||
elements.sort(key=lambda elem: elem.title)
|
||||
|
||||
tasks: List[Awaitable[None]] = []
|
||||
for elem in elements:
|
||||
tasks.append(asyncio.create_task(self._download_forum_thread(cl.path, elem)))
|
||||
|
||||
# And execute them
|
||||
await self.gather(tasks)
|
||||
|
||||
@anoncritical
|
||||
@_iorepeat(3, "saving forum thread")
|
||||
async def _download_forum_thread(
|
||||
self,
|
||||
parent_path: PurePath,
|
||||
element: IliasForumThread,
|
||||
) -> None:
|
||||
path = parent_path / (_sanitize_path_name(element.title) + ".html")
|
||||
maybe_dl = await self.download(path, mtime=element.mtime)
|
||||
if not maybe_dl:
|
||||
return
|
||||
|
||||
async with maybe_dl as (bar, sink):
|
||||
content = element.title_tag.prettify()
|
||||
content += element.content_tag.prettify()
|
||||
sink.file.write(content.encode("utf-8"))
|
||||
sink.done()
|
||||
|
||||
async def _get_page(self, url: str) -> BeautifulSoup:
|
||||
auth_id = await self._current_auth_id()
|
||||
async with self.session.get(url) as request:
|
||||
soup = soupify(await request.read())
|
||||
if self._is_logged_in(soup):
|
||||
return soup
|
||||
|
||||
# We weren't authenticated, so try to do that
|
||||
await self.authenticate(auth_id)
|
||||
|
||||
# Retry once after authenticating. If this fails, we will die.
|
||||
async with self.session.get(url) as request:
|
||||
soup = soupify(await request.read())
|
||||
if self._is_logged_in(soup):
|
||||
return soup
|
||||
raise CrawlError("get_page failed even after authenticating")
|
||||
|
||||
async def _post_authenticated(
|
||||
self,
|
||||
url: str,
|
||||
data: dict[str, Union[str, List[str]]]
|
||||
) -> BeautifulSoup:
|
||||
auth_id = await self._current_auth_id()
|
||||
|
||||
form_data = aiohttp.FormData()
|
||||
for key, val in data.items():
|
||||
form_data.add_field(key, val)
|
||||
|
||||
async with self.session.post(url, data=form_data(), allow_redirects=False) as request:
|
||||
if request.status == 200:
|
||||
return await request.read()
|
||||
|
||||
# We weren't authenticated, so try to do that
|
||||
await self.authenticate(auth_id)
|
||||
|
||||
# Retry once after authenticating. If this fails, we will die.
|
||||
async with self.session.post(url, data=data, allow_redirects=False) as request:
|
||||
if request.status == 200:
|
||||
return await request.read()
|
||||
raise CrawlError("post_authenticated failed even after authenticating")
|
||||
|
||||
# We repeat this as the login method in shibboleth doesn't handle I/O errors.
|
||||
# Shibboleth is quite reliable as well, the repeat is likely not critical here.
|
||||
@ _iorepeat(3, "Login", failure_is_error=True)
|
||||
async def _authenticate(self) -> None:
|
||||
await self._shibboleth_login.login(self.session)
|
||||
|
||||
@ staticmethod
|
||||
def _is_logged_in(soup: BeautifulSoup) -> bool:
|
||||
# Normal ILIAS pages
|
||||
mainbar: Optional[Tag] = soup.find(class_="il-maincontrols-metabar")
|
||||
if mainbar is not None:
|
||||
login_button = mainbar.find(attrs={"href": lambda x: x and "login.php" in x})
|
||||
shib_login = soup.find(id="button_shib_login")
|
||||
return not login_button and not shib_login
|
||||
|
||||
# Personal Desktop
|
||||
if soup.find("a", attrs={"href": lambda x: x and "block_type=pditems" in x}):
|
||||
return True
|
||||
|
||||
# Video listing embeds do not have complete ILIAS html. Try to match them by
|
||||
# their video listing table
|
||||
video_table = soup.find(
|
||||
recursive=True,
|
||||
name="table",
|
||||
attrs={"id": lambda x: x is not None and x.startswith("tbl_xoct")}
|
||||
)
|
||||
if video_table is not None:
|
||||
return True
|
||||
# The individual video player wrapper page has nothing of the above.
|
||||
# Match it by its playerContainer.
|
||||
if soup.select_one("#playerContainer") is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class KitShibbolethLogin:
|
||||
"""
|
||||
Login via KIT's shibboleth system.
|
||||
"""
|
||||
|
||||
def __init__(self, authenticator: Authenticator, tfa_authenticator: Optional[Authenticator]) -> None:
|
||||
self._auth = authenticator
|
||||
self._tfa_auth = tfa_authenticator
|
||||
|
||||
async def login(self, sess: aiohttp.ClientSession) -> None:
|
||||
"""
|
||||
Performs the ILIAS Shibboleth authentication dance and saves the login
|
||||
cookies it receieves.
|
||||
|
||||
This function should only be called whenever it is detected that you're
|
||||
not logged in. The cookies obtained should be good for a few minutes,
|
||||
maybe even an hour or two.
|
||||
"""
|
||||
|
||||
# Equivalent: Click on "Mit KIT-Account anmelden" button in
|
||||
# https://ilias.studium.kit.edu/login.php
|
||||
url = f"{_ILIAS_URL}/shib_login.php"
|
||||
data = {
|
||||
"sendLogin": "1",
|
||||
"idp_selection": "https://idp.scc.kit.edu/idp/shibboleth",
|
||||
"il_target": "",
|
||||
"home_organization_selection": "Weiter",
|
||||
}
|
||||
soup: Union[BeautifulSoup, KitShibbolethBackgroundLoginSuccessful] = await _shib_post(sess, url, data)
|
||||
|
||||
if isinstance(soup, KitShibbolethBackgroundLoginSuccessful):
|
||||
return
|
||||
|
||||
# Attempt to login using credentials, if necessary
|
||||
while not self._login_successful(soup):
|
||||
# Searching the form here so that this fails before asking for
|
||||
# credentials rather than after asking.
|
||||
form = soup.find("form", {"class": "full content", "method": "post"})
|
||||
action = form["action"]
|
||||
|
||||
csrf_token = form.find("input", {"name": "csrf_token"})["value"]
|
||||
|
||||
# Equivalent: Enter credentials in
|
||||
# https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
|
||||
url = "https://idp.scc.kit.edu" + action
|
||||
username, password = await self._auth.credentials()
|
||||
data = {
|
||||
"_eventId_proceed": "",
|
||||
"j_username": username,
|
||||
"j_password": password,
|
||||
"csrf_token": csrf_token
|
||||
}
|
||||
soup = await _post(sess, url, data)
|
||||
|
||||
if soup.find(id="attributeRelease"):
|
||||
raise CrawlError(
|
||||
"ILIAS Shibboleth entitlements changed! "
|
||||
"Please log in once in your browser and review them"
|
||||
)
|
||||
|
||||
if self._tfa_required(soup):
|
||||
soup = await self._authenticate_tfa(sess, soup)
|
||||
|
||||
if not self._login_successful(soup):
|
||||
self._auth.invalidate_credentials()
|
||||
|
||||
# Equivalent: Being redirected via JS automatically
|
||||
# (or clicking "Continue" if you have JS disabled)
|
||||
relay_state = soup.find("input", {"name": "RelayState"})
|
||||
saml_response = soup.find("input", {"name": "SAMLResponse"})
|
||||
url = f"{_ILIAS_URL}/Shibboleth.sso/SAML2/POST"
|
||||
data = { # using the info obtained in the while loop above
|
||||
"RelayState": relay_state["value"],
|
||||
"SAMLResponse": saml_response["value"],
|
||||
}
|
||||
await sess.post(url, data=data)
|
||||
|
||||
async def _authenticate_tfa(
|
||||
self,
|
||||
session: aiohttp.ClientSession,
|
||||
soup: BeautifulSoup
|
||||
) -> BeautifulSoup:
|
||||
if not self._tfa_auth:
|
||||
self._tfa_auth = TfaAuthenticator("ilias-anon-tfa")
|
||||
|
||||
tfa_token = await self._tfa_auth.password()
|
||||
|
||||
# Searching the form here so that this fails before asking for
|
||||
# credentials rather than after asking.
|
||||
form = soup.find("form", {"method": "post"})
|
||||
action = form["action"]
|
||||
csrf_token = form.find("input", {"name": "csrf_token"})["value"]
|
||||
|
||||
# Equivalent: Enter token in
|
||||
# https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
|
||||
url = "https://idp.scc.kit.edu" + action
|
||||
data = {
|
||||
"_eventId_proceed": "",
|
||||
"j_tokenNumber": tfa_token,
|
||||
"csrf_token": csrf_token
|
||||
}
|
||||
return await _post(session, url, data)
|
||||
|
||||
@staticmethod
|
||||
def _login_successful(soup: BeautifulSoup) -> bool:
|
||||
relay_state = soup.find("input", {"name": "RelayState"})
|
||||
saml_response = soup.find("input", {"name": "SAMLResponse"})
|
||||
return relay_state is not None and saml_response is not None
|
||||
|
||||
@staticmethod
|
||||
def _tfa_required(soup: BeautifulSoup) -> bool:
|
||||
return soup.find(id="j_tokenNumber") is not None
|
||||
|
||||
|
||||
async def _post(session: aiohttp.ClientSession, url: str, data: Any) -> BeautifulSoup:
|
||||
async with session.post(url, data=data) as response:
|
||||
return soupify(await response.read())
|
||||
|
||||
|
||||
async def _shib_post(
|
||||
session: aiohttp.ClientSession,
|
||||
url: str,
|
||||
data: Any
|
||||
) -> Union[BeautifulSoup, KitShibbolethBackgroundLoginSuccessful]:
|
||||
"""
|
||||
aiohttp unescapes '/' and ':' in URL query parameters which is not RFC compliant and rejected
|
||||
by Shibboleth. Thanks a lot. So now we unroll the requests manually, parse location headers and
|
||||
build encoded URL objects ourselves... Who thought mangling location header was a good idea??
|
||||
"""
|
||||
log.explain_topic("Shib login POST")
|
||||
async with session.post(url, data=data, allow_redirects=False) as response:
|
||||
location = response.headers.get("location")
|
||||
log.explain(f"Got location {location!r}")
|
||||
if not location:
|
||||
raise CrawlWarning(f"Login failed (1), no location header present at {url}")
|
||||
correct_url = yarl.URL(location, encoded=True)
|
||||
log.explain(f"Corrected location to {correct_url!r}")
|
||||
|
||||
if str(correct_url).startswith(_ILIAS_URL):
|
||||
log.explain("ILIAS recognized our shib token and logged us in in the background, returning")
|
||||
return KitShibbolethBackgroundLoginSuccessful()
|
||||
|
||||
async with session.get(correct_url, allow_redirects=False) as response:
|
||||
location = response.headers.get("location")
|
||||
log.explain(f"Redirected to {location!r} with status {response.status}")
|
||||
# If shib still still has a valid session, it will directly respond to the request
|
||||
if location is None:
|
||||
log.explain("Shib recognized us, returning its response directly")
|
||||
return soupify(await response.read())
|
||||
|
||||
as_yarl = yarl.URL(response.url)
|
||||
# Probably not needed anymore, but might catch a few weird situations with a nicer message
|
||||
if not location or not as_yarl.host:
|
||||
raise CrawlWarning(f"Login failed (2), no location header present at {correct_url}")
|
||||
|
||||
correct_url = yarl.URL.build(
|
||||
scheme=as_yarl.scheme,
|
||||
host=as_yarl.host,
|
||||
path=location,
|
||||
encoded=True
|
||||
)
|
||||
log.explain(f"Corrected location to {correct_url!r}")
|
||||
|
||||
async with session.get(correct_url, allow_redirects=False) as response:
|
||||
return soupify(await response.read())
|
||||
|
||||
@ -1,128 +0,0 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
import aiohttp
|
||||
import yarl
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from ...auth import Authenticator, TfaAuthenticator
|
||||
from ...logging import log
|
||||
from ...utils import soupify
|
||||
from ..crawler import CrawlError
|
||||
|
||||
|
||||
class ShibbolethLogin:
|
||||
"""
|
||||
Login via shibboleth system.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, ilias_url: str, authenticator: Authenticator, tfa_authenticator: Optional[Authenticator]
|
||||
) -> None:
|
||||
self._ilias_url = ilias_url
|
||||
self._auth = authenticator
|
||||
self._tfa_auth = tfa_authenticator
|
||||
|
||||
async def login(self, sess: aiohttp.ClientSession) -> None:
|
||||
"""
|
||||
Performs the ILIAS Shibboleth authentication dance and saves the login
|
||||
cookies it receieves.
|
||||
|
||||
This function should only be called whenever it is detected that you're
|
||||
not logged in. The cookies obtained should be good for a few minutes,
|
||||
maybe even an hour or two.
|
||||
"""
|
||||
|
||||
# Equivalent: Click on "Mit KIT-Account anmelden" button in
|
||||
# https://ilias.studium.kit.edu/login.php
|
||||
url = f"{self._ilias_url}/shib_login.php"
|
||||
async with sess.get(url) as response:
|
||||
shib_url = response.url
|
||||
if str(shib_url).startswith(self._ilias_url):
|
||||
log.explain(
|
||||
"ILIAS recognized our shib token and logged us in in the background, returning"
|
||||
)
|
||||
return
|
||||
soup: BeautifulSoup = soupify(await response.read())
|
||||
|
||||
# Attempt to login using credentials, if necessary
|
||||
while not self._login_successful(soup):
|
||||
# Searching the form here so that this fails before asking for
|
||||
# credentials rather than after asking.
|
||||
form = soup.find("form", {"method": "post"})
|
||||
action = form["action"]
|
||||
|
||||
# Equivalent: Enter credentials in
|
||||
# https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
|
||||
url = str(shib_url.origin()) + action
|
||||
username, password = await self._auth.credentials()
|
||||
data = {
|
||||
"_eventId_proceed": "",
|
||||
"j_username": username,
|
||||
"j_password": password,
|
||||
}
|
||||
if csrf_token_input := form.find("input", {"name": "csrf_token"}):
|
||||
data["csrf_token"] = csrf_token_input["value"]
|
||||
soup = await _post(sess, url, data)
|
||||
|
||||
if soup.find(id="attributeRelease"):
|
||||
raise CrawlError(
|
||||
"ILIAS Shibboleth entitlements changed! "
|
||||
"Please log in once in your browser and review them"
|
||||
)
|
||||
|
||||
if self._tfa_required(soup):
|
||||
soup = await self._authenticate_tfa(sess, soup, shib_url)
|
||||
|
||||
if not self._login_successful(soup):
|
||||
self._auth.invalidate_credentials()
|
||||
|
||||
# Equivalent: Being redirected via JS automatically
|
||||
# (or clicking "Continue" if you have JS disabled)
|
||||
relay_state = soup.find("input", {"name": "RelayState"})
|
||||
saml_response = soup.find("input", {"name": "SAMLResponse"})
|
||||
url = form = soup.find("form", {"method": "post"})["action"]
|
||||
data = { # using the info obtained in the while loop above
|
||||
"RelayState": relay_state["value"],
|
||||
"SAMLResponse": saml_response["value"],
|
||||
}
|
||||
await sess.post(url, data=data)
|
||||
|
||||
async def _authenticate_tfa(
|
||||
self, session: aiohttp.ClientSession, soup: BeautifulSoup, shib_url: yarl.URL
|
||||
) -> BeautifulSoup:
|
||||
if not self._tfa_auth:
|
||||
self._tfa_auth = TfaAuthenticator("ilias-anon-tfa")
|
||||
|
||||
tfa_token = await self._tfa_auth.password()
|
||||
|
||||
# Searching the form here so that this fails before asking for
|
||||
# credentials rather than after asking.
|
||||
form = soup.find("form", {"method": "post"})
|
||||
action = form["action"]
|
||||
|
||||
# Equivalent: Enter token in
|
||||
# https://idp.scc.kit.edu/idp/profile/SAML2/Redirect/SSO
|
||||
url = str(shib_url.origin()) + action
|
||||
username, password = await self._auth.credentials()
|
||||
data = {
|
||||
"_eventId_proceed": "",
|
||||
"j_tokenNumber": tfa_token,
|
||||
}
|
||||
if csrf_token_input := form.find("input", {"name": "csrf_token"}):
|
||||
data["csrf_token"] = csrf_token_input["value"]
|
||||
return await _post(session, url, data)
|
||||
|
||||
@staticmethod
|
||||
def _login_successful(soup: BeautifulSoup) -> bool:
|
||||
relay_state = soup.find("input", {"name": "RelayState"})
|
||||
saml_response = soup.find("input", {"name": "SAMLResponse"})
|
||||
return relay_state is not None and saml_response is not None
|
||||
|
||||
@staticmethod
|
||||
def _tfa_required(soup: BeautifulSoup) -> bool:
|
||||
return soup.find(id="j_tokenNumber") is not None
|
||||
|
||||
|
||||
async def _post(session: aiohttp.ClientSession, url: str, data: Any) -> BeautifulSoup:
|
||||
async with session.post(url, data=data) as response:
|
||||
return soupify(await response.read())
|
||||
@ -1,9 +1,8 @@
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import PurePath
|
||||
from typing import Any, Awaitable, Generator, Iterable, List, Optional, Pattern, Tuple, Union
|
||||
from typing import List, Optional, Pattern, Set, Tuple, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
@ -32,24 +31,24 @@ class KitIpdCrawlerSection(HttpCrawlerSection):
|
||||
return re.compile(regex)
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(unsafe_hash=True)
|
||||
class KitIpdFile:
|
||||
name: str
|
||||
url: str
|
||||
|
||||
def explain(self) -> None:
|
||||
log.explain(f"File {self.name!r} (href={self.url!r})")
|
||||
|
||||
|
||||
@dataclass
|
||||
class KitIpdFolder:
|
||||
name: str
|
||||
entries: List[Union[KitIpdFile, "KitIpdFolder"]]
|
||||
files: List[KitIpdFile]
|
||||
|
||||
def explain(self) -> None:
|
||||
log.explain_topic(f"Folder {self.name!r}")
|
||||
for entry in self.entries:
|
||||
entry.explain()
|
||||
for file in self.files:
|
||||
log.explain(f"File {file.name!r} (href={file.url!r})")
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return self.name.__hash__()
|
||||
|
||||
|
||||
class KitIpdCrawler(HttpCrawler):
|
||||
@ -65,91 +64,71 @@ class KitIpdCrawler(HttpCrawler):
|
||||
self._file_regex = section.link_regex()
|
||||
|
||||
async def _run(self) -> None:
|
||||
maybe_cl = await self.crawl(PurePath("."))
|
||||
if not maybe_cl:
|
||||
cl = await self.crawl(PurePath("."))
|
||||
if not cl:
|
||||
return
|
||||
|
||||
tasks: List[Awaitable[None]] = []
|
||||
|
||||
async with maybe_cl:
|
||||
async with cl:
|
||||
for item in await self._fetch_items():
|
||||
item.explain()
|
||||
if isinstance(item, KitIpdFolder):
|
||||
tasks.append(self._crawl_folder(PurePath("."), item))
|
||||
await self._crawl_folder(item)
|
||||
else:
|
||||
log.explain_topic(f"Orphan file {item.name!r} (href={item.url!r})")
|
||||
log.explain("Attributing it to root folder")
|
||||
# do this here to at least be sequential and not parallel (rate limiting is hard, as the
|
||||
# crawl abstraction does not hold for these requests)
|
||||
etag, mtime = await self._request_resource_version(item.url)
|
||||
tasks.append(self._download_file(PurePath("."), item, etag, mtime))
|
||||
# Orphan files are placed in the root folder
|
||||
await self._download_file(PurePath("."), item)
|
||||
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _crawl_folder(self, parent: PurePath, folder: KitIpdFolder) -> None:
|
||||
path = parent / folder.name
|
||||
async def _crawl_folder(self, folder: KitIpdFolder) -> None:
|
||||
path = PurePath(folder.name)
|
||||
if not await self.crawl(path):
|
||||
return
|
||||
|
||||
tasks = []
|
||||
for entry in folder.entries:
|
||||
if isinstance(entry, KitIpdFolder):
|
||||
tasks.append(self._crawl_folder(path, entry))
|
||||
else:
|
||||
# do this here to at least be sequential and not parallel (rate limiting is hard, as the crawl
|
||||
# abstraction does not hold for these requests)
|
||||
etag, mtime = await self._request_resource_version(entry.url)
|
||||
tasks.append(self._download_file(path, entry, etag, mtime))
|
||||
for file in folder.files:
|
||||
await self._download_file(path, file)
|
||||
|
||||
await self.gather(tasks)
|
||||
|
||||
async def _download_file(
|
||||
self,
|
||||
parent: PurePath,
|
||||
file: KitIpdFile,
|
||||
etag: Optional[str],
|
||||
mtime: Optional[datetime]
|
||||
) -> None:
|
||||
async def _download_file(self, parent: PurePath, file: KitIpdFile) -> None:
|
||||
element_path = parent / file.name
|
||||
|
||||
prev_etag = self._get_previous_etag_from_report(element_path)
|
||||
etag_differs = None if prev_etag is None else prev_etag != etag
|
||||
|
||||
maybe_dl = await self.download(element_path, etag_differs=etag_differs, mtime=mtime)
|
||||
if not maybe_dl:
|
||||
# keep storing the known file's etag
|
||||
if prev_etag:
|
||||
self._add_etag_to_report(element_path, prev_etag)
|
||||
dl = await self.download(element_path)
|
||||
if not dl:
|
||||
return
|
||||
|
||||
async with maybe_dl as (bar, sink):
|
||||
await self._stream_from_url(file.url, element_path, sink, bar)
|
||||
async with dl as (bar, sink):
|
||||
await self._stream_from_url(file.url, sink, bar)
|
||||
|
||||
async def _fetch_items(self) -> Iterable[Union[KitIpdFile, KitIpdFolder]]:
|
||||
page, url = await self.get_page()
|
||||
async def _fetch_items(self) -> Set[Union[KitIpdFile, KitIpdFolder]]:
|
||||
page, url = await self._get_page()
|
||||
elements: List[Tag] = self._find_file_links(page)
|
||||
items: Set[Union[KitIpdFile, KitIpdFolder]] = set()
|
||||
|
||||
# do not add unnecessary nesting for a single <h1> heading
|
||||
drop_h1: bool = len(page.find_all(name="h1")) <= 1
|
||||
|
||||
folder_tree: KitIpdFolder = KitIpdFolder(".", [])
|
||||
for element in elements:
|
||||
parent = HttpCrawler.get_folder_structure_from_heading_hierarchy(element, drop_h1)
|
||||
folder_label = self._find_folder_label(element)
|
||||
if folder_label:
|
||||
folder = self._extract_folder(folder_label, url)
|
||||
if folder not in items:
|
||||
items.add(folder)
|
||||
folder.explain()
|
||||
else:
|
||||
file = self._extract_file(element, url)
|
||||
items.add(file)
|
||||
log.explain_topic(f"Orphan file {file.name!r} (href={file.url!r})")
|
||||
log.explain("Attributing it to root folder")
|
||||
|
||||
current_folder: KitIpdFolder = folder_tree
|
||||
for folder_name in parent.parts:
|
||||
# helps the type checker to verify that current_folder is indeed a folder
|
||||
def subfolders() -> Generator[KitIpdFolder, Any, None]:
|
||||
return (entry for entry in current_folder.entries if isinstance(entry, KitIpdFolder))
|
||||
return items
|
||||
|
||||
if not any(entry.name == folder_name for entry in subfolders()):
|
||||
current_folder.entries.append(KitIpdFolder(folder_name, []))
|
||||
current_folder = next(entry for entry in subfolders() if entry.name == folder_name)
|
||||
def _extract_folder(self, folder_tag: Tag, url: str) -> KitIpdFolder:
|
||||
files: List[KitIpdFile] = []
|
||||
name = folder_tag.getText().strip()
|
||||
|
||||
current_folder.entries.append(file)
|
||||
container: Tag = folder_tag.findNextSibling(name="table")
|
||||
for link in self._find_file_links(container):
|
||||
files.append(self._extract_file(link, url))
|
||||
|
||||
return folder_tree.entries
|
||||
return KitIpdFolder(name, files)
|
||||
|
||||
@staticmethod
|
||||
def _find_folder_label(file_link: Tag) -> Optional[Tag]:
|
||||
enclosing_table: Tag = file_link.findParent(name="table")
|
||||
if enclosing_table is None:
|
||||
return None
|
||||
return enclosing_table.findPreviousSibling(name=re.compile("^h[1-6]$"))
|
||||
|
||||
def _extract_file(self, link: Tag, url: str) -> KitIpdFile:
|
||||
url = self._abs_url_from_link(url, link)
|
||||
@ -162,7 +141,7 @@ class KitIpdCrawler(HttpCrawler):
|
||||
def _abs_url_from_link(self, url: str, link_tag: Tag) -> str:
|
||||
return urljoin(url, link_tag.get("href"))
|
||||
|
||||
async def _stream_from_url(self, url: str, path: PurePath, sink: FileSink, bar: ProgressBar) -> None:
|
||||
async def _stream_from_url(self, url: str, sink: FileSink, bar: ProgressBar) -> None:
|
||||
async with self.session.get(url, allow_redirects=False) as resp:
|
||||
if resp.status == 403:
|
||||
raise CrawlError("Received a 403. Are you within the KIT network/VPN?")
|
||||
@ -175,14 +154,12 @@ class KitIpdCrawler(HttpCrawler):
|
||||
|
||||
sink.done()
|
||||
|
||||
self._add_etag_to_report(path, resp.headers.get("ETag"))
|
||||
async def _get_page(self) -> Tuple[BeautifulSoup, str]:
|
||||
response = self.session.get(self._url)
|
||||
|
||||
async def get_page(self) -> Tuple[BeautifulSoup, str]:
|
||||
async with self.session.get(self._url) as request:
|
||||
# The web page for Algorithmen für Routenplanung contains some
|
||||
# weird comments that beautifulsoup doesn't parse correctly. This
|
||||
# hack enables those pages to be crawled, and should hopefully not
|
||||
# cause issues on other pages.
|
||||
content = (await request.read()).decode("utf-8")
|
||||
content = re.sub(r"<!--.*?-->", "", content)
|
||||
content = re.sub(r"<!--.*?-->", "", response.text)
|
||||
return soupify(content.encode("utf-8")), str(request.url)
|
||||
|
||||
@ -71,8 +71,6 @@ class LocalCrawler(Crawler):
|
||||
if not cl:
|
||||
return
|
||||
|
||||
tasks = []
|
||||
|
||||
async with cl:
|
||||
await asyncio.sleep(random.uniform(
|
||||
0.5 * self._crawl_delay,
|
||||
@ -81,9 +79,7 @@ class LocalCrawler(Crawler):
|
||||
|
||||
for child in path.iterdir():
|
||||
pure_child = cl.path / child.name
|
||||
tasks.append(self._crawl_path(child, pure_child))
|
||||
|
||||
await self.gather(tasks)
|
||||
await self._crawl_path(child, pure_child)
|
||||
|
||||
async def _crawl_file(self, path: Path, pure: PurePath) -> None:
|
||||
stat = path.stat()
|
||||
|
||||
@ -14,7 +14,7 @@ def name_variants(path: PurePath) -> Iterator[PurePath]:
|
||||
|
||||
|
||||
class Deduplicator:
|
||||
FORBIDDEN_CHARS = '<>:"/\\|?*' + "".join([chr(i) for i in range(0, 32)])
|
||||
FORBIDDEN_CHARS = '<>:"/\\|?*'
|
||||
FORBIDDEN_NAMES = {
|
||||
"CON", "PRN", "AUX", "NUL",
|
||||
"COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
|
||||
|
||||
@ -1,97 +0,0 @@
|
||||
import asyncio
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class Slot:
|
||||
active: bool = False
|
||||
last_left: Optional[float] = None
|
||||
|
||||
|
||||
class Limiter:
|
||||
def __init__(
|
||||
self,
|
||||
task_limit: int,
|
||||
download_limit: int,
|
||||
task_delay: float
|
||||
):
|
||||
if task_limit <= 0:
|
||||
raise ValueError("task limit must be at least 1")
|
||||
if download_limit <= 0:
|
||||
raise ValueError("download limit must be at least 1")
|
||||
if download_limit > task_limit:
|
||||
raise ValueError("download limit can't be greater than task limit")
|
||||
if task_delay < 0:
|
||||
raise ValueError("Task delay must not be negative")
|
||||
|
||||
self._slots = [Slot() for _ in range(task_limit)]
|
||||
self._downloads = download_limit
|
||||
self._delay = task_delay
|
||||
|
||||
self._condition = asyncio.Condition()
|
||||
|
||||
def _acquire_slot(self) -> Optional[Slot]:
|
||||
for slot in self._slots:
|
||||
if not slot.active:
|
||||
slot.active = True
|
||||
return slot
|
||||
|
||||
return None
|
||||
|
||||
async def _wait_for_slot_delay(self, slot: Slot) -> None:
|
||||
if slot.last_left is not None:
|
||||
delay = slot.last_left + self._delay - time.time()
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
def _release_slot(self, slot: Slot) -> None:
|
||||
slot.last_left = time.time()
|
||||
slot.active = False
|
||||
|
||||
@asynccontextmanager
|
||||
async def limit_crawl(self) -> AsyncIterator[None]:
|
||||
slot: Slot
|
||||
async with self._condition:
|
||||
while True:
|
||||
if found_slot := self._acquire_slot():
|
||||
slot = found_slot
|
||||
break
|
||||
await self._condition.wait()
|
||||
|
||||
await self._wait_for_slot_delay(slot)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
async with self._condition:
|
||||
self._release_slot(slot)
|
||||
self._condition.notify_all()
|
||||
|
||||
@asynccontextmanager
|
||||
async def limit_download(self) -> AsyncIterator[None]:
|
||||
slot: Slot
|
||||
async with self._condition:
|
||||
while True:
|
||||
if self._downloads <= 0:
|
||||
await self._condition.wait()
|
||||
continue
|
||||
|
||||
if found_slot := self._acquire_slot():
|
||||
slot = found_slot
|
||||
self._downloads -= 1
|
||||
break
|
||||
|
||||
await self._condition.wait()
|
||||
|
||||
await self._wait_for_slot_delay(slot)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
async with self._condition:
|
||||
self._release_slot(slot)
|
||||
self._downloads += 1
|
||||
self._condition.notify_all()
|
||||
@ -59,7 +59,6 @@ class Log:
|
||||
# Whether different parts of the output are enabled or disabled
|
||||
self.output_explain = False
|
||||
self.output_status = True
|
||||
self.output_not_deleted = True
|
||||
self.output_report = True
|
||||
|
||||
def _update_live(self) -> None:
|
||||
@ -208,17 +207,6 @@ directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
|
||||
action = escape(f"{action:<{self.STATUS_WIDTH}}")
|
||||
self.print(f"{style}{action}[/] {escape(text)} {suffix}")
|
||||
|
||||
def not_deleted(self, style: str, action: str, text: str, suffix: str = "") -> None:
|
||||
"""
|
||||
Print a message for a local only file that wasn't
|
||||
deleted while crawling. Allows markup in the "style"
|
||||
argument which will be applied to the "action" string.
|
||||
"""
|
||||
|
||||
if self.output_status and self.output_not_deleted:
|
||||
action = escape(f"{action:<{self.STATUS_WIDTH}}")
|
||||
self.print(f"{style}{action}[/] {escape(text)} {suffix}")
|
||||
|
||||
def report(self, text: str) -> None:
|
||||
"""
|
||||
Print a report after crawling. Allows markup.
|
||||
@ -227,14 +215,6 @@ directly or as a GitHub issue: https://github.com/Garmelon/PFERD/issues/new
|
||||
if self.output_report:
|
||||
self.print(text)
|
||||
|
||||
def report_not_deleted(self, text: str) -> None:
|
||||
"""
|
||||
Print a report for a local only file that wasn't deleted after crawling. Allows markup.
|
||||
"""
|
||||
|
||||
if self.output_report and self.output_not_deleted:
|
||||
self.print(text)
|
||||
|
||||
@contextmanager
|
||||
def _bar(
|
||||
self,
|
||||
|
||||
@ -44,7 +44,6 @@ class OnConflict(Enum):
|
||||
LOCAL_FIRST = "local-first"
|
||||
REMOTE_FIRST = "remote-first"
|
||||
NO_DELETE = "no-delete"
|
||||
NO_DELETE_PROMPT_OVERWRITE = "no-delete-prompt-overwrite"
|
||||
|
||||
@staticmethod
|
||||
def from_string(string: str) -> "OnConflict":
|
||||
@ -52,12 +51,11 @@ class OnConflict(Enum):
|
||||
return OnConflict(string)
|
||||
except ValueError:
|
||||
raise ValueError("must be one of 'prompt', 'local-first',"
|
||||
" 'remote-first', 'no-delete', 'no-delete-prompt-overwrite'")
|
||||
" 'remote-first', 'no-delete'")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Heuristics:
|
||||
etag_differs: Optional[bool]
|
||||
mtime: Optional[datetime]
|
||||
|
||||
|
||||
@ -234,16 +232,8 @@ class OutputDirectory:
|
||||
|
||||
remote_newer = None
|
||||
|
||||
# ETag should be a more reliable indicator than mtime, so we check it first
|
||||
if heuristics.etag_differs is not None:
|
||||
remote_newer = heuristics.etag_differs
|
||||
if remote_newer:
|
||||
log.explain("Remote file's entity tag differs")
|
||||
else:
|
||||
log.explain("Remote file's entity tag is the same")
|
||||
|
||||
# Python on Windows crashes when faced with timestamps around the unix epoch
|
||||
if remote_newer is None and heuristics.mtime and (os.name != "nt" or heuristics.mtime.year > 1970):
|
||||
if heuristics.mtime and (os.name != "nt" or heuristics.mtime.year > 1970):
|
||||
mtime = heuristics.mtime
|
||||
remote_newer = mtime.timestamp() > stat.st_mtime
|
||||
if remote_newer:
|
||||
@ -274,7 +264,7 @@ class OutputDirectory:
|
||||
on_conflict: OnConflict,
|
||||
path: PurePath,
|
||||
) -> bool:
|
||||
if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
|
||||
if on_conflict == OnConflict.PROMPT:
|
||||
async with log.exclusive_output():
|
||||
prompt = f"Replace {fmt_path(path)} with remote file?"
|
||||
return await prompt_yes_no(prompt, default=False)
|
||||
@ -293,7 +283,7 @@ class OutputDirectory:
|
||||
on_conflict: OnConflict,
|
||||
path: PurePath,
|
||||
) -> bool:
|
||||
if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
|
||||
if on_conflict == OnConflict.PROMPT:
|
||||
async with log.exclusive_output():
|
||||
prompt = f"Recursively delete {fmt_path(path)} and replace with remote file?"
|
||||
return await prompt_yes_no(prompt, default=False)
|
||||
@ -313,7 +303,7 @@ class OutputDirectory:
|
||||
path: PurePath,
|
||||
parent: PurePath,
|
||||
) -> bool:
|
||||
if on_conflict in {OnConflict.PROMPT, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
|
||||
if on_conflict == OnConflict.PROMPT:
|
||||
async with log.exclusive_output():
|
||||
prompt = f"Delete {fmt_path(parent)} so remote file {fmt_path(path)} can be downloaded?"
|
||||
return await prompt_yes_no(prompt, default=False)
|
||||
@ -340,7 +330,7 @@ class OutputDirectory:
|
||||
return False
|
||||
elif on_conflict == OnConflict.REMOTE_FIRST:
|
||||
return True
|
||||
elif on_conflict in {OnConflict.NO_DELETE, OnConflict.NO_DELETE_PROMPT_OVERWRITE}:
|
||||
elif on_conflict == OnConflict.NO_DELETE:
|
||||
return False
|
||||
|
||||
# This should never be reached
|
||||
@ -375,8 +365,6 @@ class OutputDirectory:
|
||||
self,
|
||||
remote_path: PurePath,
|
||||
path: PurePath,
|
||||
*,
|
||||
etag_differs: Optional[bool] = None,
|
||||
mtime: Optional[datetime] = None,
|
||||
redownload: Optional[Redownload] = None,
|
||||
on_conflict: Optional[OnConflict] = None,
|
||||
@ -386,7 +374,7 @@ class OutputDirectory:
|
||||
MarkConflictError.
|
||||
"""
|
||||
|
||||
heuristics = Heuristics(etag_differs, mtime)
|
||||
heuristics = Heuristics(mtime)
|
||||
redownload = self._redownload if redownload is None else redownload
|
||||
on_conflict = self._on_conflict if on_conflict is None else on_conflict
|
||||
local_path = self.resolve(path)
|
||||
@ -507,7 +495,7 @@ class OutputDirectory:
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
log.not_deleted("[bold bright_magenta]", "Not deleted", fmt_path(pure))
|
||||
log.status("[bold bright_magenta]", "Not deleted", fmt_path(pure))
|
||||
self._report.not_delete_file(pure)
|
||||
|
||||
def load_prev_report(self) -> None:
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
from pathlib import Path, PurePath
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
from rich.markup import escape
|
||||
|
||||
@ -169,26 +168,19 @@ class Pferd:
|
||||
log.report("")
|
||||
log.report(f"[bold bright_cyan]Report[/] for {escape(name)}")
|
||||
|
||||
def fmt_path_link(relative_path: PurePath) -> str:
|
||||
# We need to URL-encode the path because it might contain spaces or special characters
|
||||
absolute_path = str(crawler.output_dir.resolve(relative_path).absolute())
|
||||
absolute_path = absolute_path.replace("\\\\?\\", "")
|
||||
link = f"file://{quote(absolute_path)}"
|
||||
return f"[link={link}]{fmt_path(relative_path)}[/link]"
|
||||
|
||||
something_changed = False
|
||||
for path in sorted(crawler.report.added_files):
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_green]Added[/] {fmt_path_link(path)}")
|
||||
log.report(f" [bold bright_green]Added[/] {fmt_path(path)}")
|
||||
for path in sorted(crawler.report.changed_files):
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_yellow]Changed[/] {fmt_path_link(path)}")
|
||||
log.report(f" [bold bright_yellow]Changed[/] {fmt_path(path)}")
|
||||
for path in sorted(crawler.report.deleted_files):
|
||||
something_changed = True
|
||||
log.report(f" [bold bright_magenta]Deleted[/] {fmt_path(path)}")
|
||||
for path in sorted(crawler.report.not_deleted_files):
|
||||
something_changed = True
|
||||
log.report_not_deleted(f" [bold bright_magenta]Not deleted[/] {fmt_path_link(path)}")
|
||||
log.report(f" [bold bright_magenta]Not deleted[/] {fmt_path(path)}")
|
||||
|
||||
for warning in crawler.report.encountered_warnings:
|
||||
something_changed = True
|
||||
|
||||
@ -110,10 +110,6 @@ class ExactReTf(Transformation):
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
named_groups: Dict[str, str] = match.groupdict()
|
||||
for name, capture in named_groups.items():
|
||||
locals_dir[name] = capture
|
||||
|
||||
result = eval(f"f{right!r}", {}, locals_dir)
|
||||
return Transformed(PurePath(result))
|
||||
|
||||
|
||||
@ -92,17 +92,32 @@ def url_set_query_params(url: str, params: Dict[str, str]) -> str:
|
||||
|
||||
|
||||
def str_path(path: PurePath) -> str:
|
||||
"""
|
||||
Turn a path into a string, in a platform-independent way.
|
||||
|
||||
This function always uses "/" as path separator, even on Windows.
|
||||
"""
|
||||
if not path.parts:
|
||||
return "."
|
||||
return "/".join(path.parts)
|
||||
|
||||
|
||||
def fmt_path(path: PurePath) -> str:
|
||||
"""
|
||||
Turn a path into a delimited string.
|
||||
|
||||
This is useful if file or directory names contain weird characters like
|
||||
newlines, leading/trailing whitespace or unprintable characters. This way,
|
||||
they are escaped and visible to the user.
|
||||
"""
|
||||
return repr(str_path(path))
|
||||
|
||||
|
||||
def fmt_real_path(path: Path) -> str:
|
||||
return repr(str(path.absolute()))
|
||||
"""
|
||||
Like fmt_path, but resolves the path before converting it to a string.
|
||||
"""
|
||||
return fmt_path(path.absolute())
|
||||
|
||||
|
||||
class ReusableAsyncContextManager(ABC, Generic[T]):
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
NAME = "PFERD"
|
||||
VERSION = "3.7.0"
|
||||
VERSION = "3.4.3"
|
||||
|
||||
11
README.md
11
README.md
@ -56,17 +56,6 @@ Also, you can download most ILIAS pages directly like this:
|
||||
$ pferd kit-ilias-web <url> <output_directory>
|
||||
```
|
||||
|
||||
PFERD supports other ILIAS instances as well, using the `ilias-web` crawler (see
|
||||
the [config section on `ilias-web`](CONFIG.md#the-ilias-web-crawler) for more
|
||||
detail on the `base-url` and `client-id` parameters):
|
||||
|
||||
```
|
||||
$ pferd ilias-web \
|
||||
--base-url https://ilias.my-university.example \
|
||||
--client-id My_University desktop \
|
||||
<output_directory>
|
||||
```
|
||||
|
||||
However, the CLI only lets you download a single thing at a time, and the
|
||||
resulting command can grow long quite quickly. Because of this, PFERD can also
|
||||
be used with a config file.
|
||||
|
||||
27
flake.lock
generated
27
flake.lock
generated
@ -1,27 +0,0 @@
|
||||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1708979614,
|
||||
"narHash": "sha256-FWLWmYojIg6TeqxSnHkKpHu5SGnFP5um1uUjH+wRV6g=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b7ee09cf5614b02d289cd86fcfa6f24d4e078c2a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-23.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
41
flake.nix
41
flake.nix
@ -1,41 +0,0 @@
|
||||
{
|
||||
description = "Tool for downloading course-related files from ILIAS";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
# Helper function to generate an attrset '{ x86_64-linux = f "x86_64-linux"; ... }'.
|
||||
forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
|
||||
in
|
||||
{
|
||||
packages = forAllSystems (system:
|
||||
let pkgs = import nixpkgs { inherit system; };
|
||||
in
|
||||
rec {
|
||||
default = pkgs.python3Packages.buildPythonApplication rec {
|
||||
pname = "pferd";
|
||||
# Performing black magic
|
||||
# Don't worry, I sacrificed enough goats for the next few years
|
||||
version = (pkgs.lib.importTOML ./PFERD/version.py).VERSION;
|
||||
format = "pyproject";
|
||||
|
||||
src = ./.;
|
||||
|
||||
nativeBuildInputs = with pkgs.python3Packages; [
|
||||
setuptools
|
||||
];
|
||||
|
||||
propagatedBuildInputs = with pkgs.python3Packages; [
|
||||
aiohttp
|
||||
beautifulsoup4
|
||||
rich
|
||||
keyring
|
||||
certifi
|
||||
];
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
11
mypy.ini
Normal file
11
mypy.ini
Normal file
@ -0,0 +1,11 @@
|
||||
[mypy]
|
||||
disallow_any_generics = True
|
||||
disallow_untyped_defs = True
|
||||
disallow_incomplete_defs = True
|
||||
no_implicit_optional = True
|
||||
warn_unused_ignores = True
|
||||
warn_unreachable = True
|
||||
show_error_context = True
|
||||
|
||||
[mypy-rich.*,bs4,keyring]
|
||||
ignore_missing_imports = True
|
||||
@ -1,42 +1,3 @@
|
||||
[build-system]
|
||||
requires = ["setuptools", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "PFERD"
|
||||
dependencies = [
|
||||
"aiohttp>=3.8.1",
|
||||
"beautifulsoup4>=4.10.0",
|
||||
"rich>=11.0.0",
|
||||
"keyring>=23.5.0",
|
||||
"certifi>=2021.10.8"
|
||||
]
|
||||
dynamic = ["version"]
|
||||
requires-python = ">=3.9"
|
||||
|
||||
[project.scripts]
|
||||
pferd = "PFERD.__main__:main"
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {attr = "PFERD.version.VERSION"}
|
||||
|
||||
[tool.flake8]
|
||||
max-line-length = 110
|
||||
|
||||
[tool.isort]
|
||||
line_length = 110
|
||||
|
||||
[tool.autopep8]
|
||||
max_line_length = 110
|
||||
in-place = true
|
||||
recursive = true
|
||||
|
||||
[tool.mypy]
|
||||
disallow_any_generics = true
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
no_implicit_optional = true
|
||||
warn_unused_ignores = true
|
||||
warn_unreachable = true
|
||||
show_error_context = true
|
||||
ignore_missing_imports = true
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import time
|
||||
import re
|
||||
from subprocess import run
|
||||
|
||||
|
||||
|
||||
@ -2,5 +2,5 @@
|
||||
|
||||
set -e
|
||||
|
||||
mypy .
|
||||
mypy PFERD
|
||||
flake8 PFERD
|
||||
|
||||
@ -2,5 +2,5 @@
|
||||
|
||||
set -e
|
||||
|
||||
autopep8 .
|
||||
isort .
|
||||
autopep8 --recursive --in-place PFERD
|
||||
isort PFERD
|
||||
|
||||
@ -13,5 +13,5 @@ pip install --upgrade setuptools
|
||||
pip install --editable .
|
||||
|
||||
# Installing tools and type hints
|
||||
pip install --upgrade mypy flake8 flake8-pyproject autopep8 isort pyinstaller
|
||||
pip install --upgrade types-chardet types-certifi
|
||||
pip install --upgrade mypy flake8 autopep8 isort pyinstaller
|
||||
mypy PFERD --install-types --non-interactive
|
||||
|
||||
24
setup.cfg
Normal file
24
setup.cfg
Normal file
@ -0,0 +1,24 @@
|
||||
[metadata]
|
||||
name = PFERD
|
||||
version = attr: PFERD.version.VERSION
|
||||
|
||||
[options]
|
||||
packages = find:
|
||||
python_requires = >=3.9
|
||||
install_requires =
|
||||
aiohttp>=3.8.1
|
||||
beautifulsoup4>=4.10.0
|
||||
rich>=11.0.0
|
||||
keyring>=23.5.0
|
||||
certifi>=2021.10.8
|
||||
requests>=2.28.1
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
pferd = PFERD.__main__:main
|
||||
|
||||
[flake8]
|
||||
max_line_length = 110
|
||||
|
||||
[isort]
|
||||
line_length = 110
|
||||
Reference in New Issue
Block a user