diff --git a/PFERD/cli/__init__.py b/PFERD/cli/__init__.py index efa8f00..c89f6f4 100644 --- a/PFERD/cli/__init__.py +++ b/PFERD/cli/__init__.py @@ -8,6 +8,7 @@ # well. from . import command_local # noqa: F401 imported but unused +from . import command_ilias_web # noqa: F401 imported but unused from . import command_kit_ilias_web # noqa: F401 imported but unused from . import command_kit_ipd # noqa: F401 imported but unused from .parser import PARSER, ParserLoadError, load_default_section # noqa: F401 imported but unused diff --git a/PFERD/cli/command_ilias_web.py b/PFERD/cli/command_ilias_web.py new file mode 100644 index 0000000..1e34668 --- /dev/null +++ b/PFERD/cli/command_ilias_web.py @@ -0,0 +1,43 @@ +import argparse +import configparser + +from .ilias_common import ilias_common_load, configure_common_group_args +from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler +from ..logging import log + +_PARSER_NAME = "ilias-web" + +SUBPARSER = SUBPARSERS.add_parser( + _PARSER_NAME, + parents=[CRAWLER_PARSER], +) + +GROUP = SUBPARSER.add_argument_group( + title=f"{_PARSER_NAME} crawler arguments", + description=f"arguments for the '{_PARSER_NAME}' crawler", +) + +GROUP.add_argument( + "--ilias-url", + type=str, + metavar="ILIAS", + help="The base url of the ilias instance" +) + +configure_common_group_args(GROUP) + + +def load(args: argparse.Namespace, parser: configparser.ConfigParser) -> None: + log.explain(f"Creating config for command '{_PARSER_NAME}'") + + parser["crawl:ilias"] = {} + section = parser["crawl:ilias"] + load_crawler(args, section) + + section["type"] = _PARSER_NAME + section["ilias_url"] = str(args.ilias_url) + + ilias_common_load(section, args, parser) + + +SUBPARSER.set_defaults(command=load) diff --git a/PFERD/cli/command_kit_ilias_web.py b/PFERD/cli/command_kit_ilias_web.py index de74fc3..bae2de1 100644 --- a/PFERD/cli/command_kit_ilias_web.py +++ b/PFERD/cli/command_kit_ilias_web.py @@ -1,120 +1,35 @@ import argparse import configparser -from pathlib import Path -from ..crawl.ilias.file_templates import Links +from .ilias_common import ilias_common_load, configure_common_group_args +from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler from ..logging import log -from .parser import (CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, ParserLoadError, load_crawler, - show_value_error) + +_PARSER_NAME = "kit-ilias-web" SUBPARSER = SUBPARSERS.add_parser( - "kit-ilias-web", + _PARSER_NAME, parents=[CRAWLER_PARSER], ) GROUP = SUBPARSER.add_argument_group( - title="kit-ilias-web crawler arguments", - description="arguments for the 'kit-ilias-web' crawler", -) -GROUP.add_argument( - "target", - type=str, - metavar="TARGET", - help="course id, 'desktop', or ILIAS URL to crawl" -) -GROUP.add_argument( - "output", - type=Path, - metavar="OUTPUT", - help="output directory" -) -GROUP.add_argument( - "--username", "-u", - type=str, - metavar="USERNAME", - help="user name for authentication" -) -GROUP.add_argument( - "--keyring", - action=BooleanOptionalAction, - help="use the system keyring to store and retrieve passwords" -) -GROUP.add_argument( - "--credential-file", - type=Path, - metavar="PATH", - help="read username and password from a credential file" -) -GROUP.add_argument( - "--links", - type=show_value_error(Links.from_string), - metavar="OPTION", - help="how to represent external links" -) -GROUP.add_argument( - "--link-redirect-delay", - type=int, - metavar="SECONDS", - help="time before 'fancy' links redirect to to their target (-1 to disable)" -) -GROUP.add_argument( - "--videos", - action=BooleanOptionalAction, - help="crawl and download videos" -) -GROUP.add_argument( - "--forums", - action=BooleanOptionalAction, - help="crawl and download forum posts" -) -GROUP.add_argument( - "--http-timeout", "-t", - type=float, - metavar="SECONDS", - help="timeout for all HTTP requests" + title=f"{_PARSER_NAME} crawler arguments", + description=f"arguments for the '{_PARSER_NAME}' crawler", ) +configure_common_group_args(GROUP) -def load( - args: argparse.Namespace, - parser: configparser.ConfigParser, -) -> None: - log.explain("Creating config for command 'kit-ilias-web'") + +def load(args: argparse.Namespace, parser: configparser.ConfigParser) -> None: + log.explain(f"Creating config for command '{_PARSER_NAME}'") parser["crawl:ilias"] = {} section = parser["crawl:ilias"] load_crawler(args, section) - section["type"] = "kit-ilias-web" - section["target"] = str(args.target) - section["output_dir"] = str(args.output) - section["auth"] = "auth:ilias" - if args.links is not None: - section["links"] = str(args.links.value) - if args.link_redirect_delay is not None: - section["link_redirect_delay"] = str(args.link_redirect_delay) - if args.videos is not None: - section["videos"] = "yes" if args.videos else "no" - if args.forums is not None: - section["forums"] = "yes" if args.forums else "no" - if args.http_timeout is not None: - section["http_timeout"] = str(args.http_timeout) + section["type"] = _PARSER_NAME - parser["auth:ilias"] = {} - auth_section = parser["auth:ilias"] - if args.credential_file is not None: - if args.username is not None: - raise ParserLoadError("--credential-file and --username can't be used together") - if args.keyring: - raise ParserLoadError("--credential-file and --keyring can't be used together") - auth_section["type"] = "credential-file" - auth_section["path"] = str(args.credential_file) - elif args.keyring: - auth_section["type"] = "keyring" - else: - auth_section["type"] = "simple" - if args.username is not None: - auth_section["username"] = args.username + ilias_common_load(section, args, parser) SUBPARSER.set_defaults(command=load) diff --git a/PFERD/cli/ilias_common.py b/PFERD/cli/ilias_common.py new file mode 100644 index 0000000..438963f --- /dev/null +++ b/PFERD/cli/ilias_common.py @@ -0,0 +1,102 @@ +import argparse +import configparser +from pathlib import Path + +from .parser import BooleanOptionalAction, ParserLoadError, show_value_error +from ..crawl.ilias.file_templates import Links + + +def configure_common_group_args(group: argparse._ArgumentGroup) -> None: + group.add_argument( + "target", + type=str, + metavar="TARGET", + help="course id, 'desktop', or ILIAS URL to crawl" + ) + group.add_argument( + "output", + type=Path, + metavar="OUTPUT", + help="output directory" + ) + group.add_argument( + "--username", "-u", + type=str, + metavar="USERNAME", + help="user name for authentication" + ) + group.add_argument( + "--keyring", + action=BooleanOptionalAction, + help="use the system keyring to store and retrieve passwords" + ) + group.add_argument( + "--credential-file", + type=Path, + metavar="PATH", + help="read username and password from a credential file" + ) + group.add_argument( + "--links", + type=show_value_error(Links.from_string), + metavar="OPTION", + help="how to represent external links" + ) + group.add_argument( + "--link-redirect-delay", + type=int, + metavar="SECONDS", + help="time before 'fancy' links redirect to to their target (-1 to disable)" + ) + group.add_argument( + "--videos", + action=BooleanOptionalAction, + help="crawl and download videos" + ) + group.add_argument( + "--forums", + action=BooleanOptionalAction, + help="crawl and download forum posts" + ) + group.add_argument( + "--http-timeout", "-t", + type=float, + metavar="SECONDS", + help="timeout for all HTTP requests" + ) + + +def ilias_common_load( + section: configparser.SectionProxy, + args: argparse.Namespace, + parser: configparser.ConfigParser, +) -> None: + section["target"] = str(args.target) + section["output_dir"] = str(args.output) + section["auth"] = "auth:ilias" + if args.links is not None: + section["links"] = str(args.links.value) + if args.link_redirect_delay is not None: + section["link_redirect_delay"] = str(args.link_redirect_delay) + if args.videos is not None: + section["videos"] = "yes" if args.videos else "no" + if args.forums is not None: + section["forums"] = "yes" if args.forums else "no" + if args.http_timeout is not None: + section["http_timeout"] = str(args.http_timeout) + + parser["auth:ilias"] = {} + auth_section = parser["auth:ilias"] + if args.credential_file is not None: + if args.username is not None: + raise ParserLoadError("--credential-file and --username can't be used together") + if args.keyring: + raise ParserLoadError("--credential-file and --keyring can't be used together") + auth_section["type"] = "credential-file" + auth_section["path"] = str(args.credential_file) + elif args.keyring: + auth_section["type"] = "keyring" + else: + auth_section["type"] = "simple" + if args.username is not None: + auth_section["username"] = args.username