add common ilias config parser

common options should be shared between KIT and generic Ilias instances

PFERD/cli/__init__.py

@@ -8,6 +8,7 @@
 # well.
 
 from . import command_local  # noqa: F401 imported but unused
+from . import command_ilias_web  # noqa: F401 imported but unused
 from . import command_kit_ilias_web  # noqa: F401 imported but unused
 from . import command_kit_ipd  # noqa: F401 imported but unused
 from .parser import PARSER, ParserLoadError, load_default_section  # noqa: F401 imported but unused

PFERD/cli/command_ilias_web.py

@@ -0,0 +1,43 @@
+import argparse
+import configparser
+
+from .ilias_common import ilias_common_load, configure_common_group_args
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
+from ..logging import log
+
+_PARSER_NAME = "ilias-web"
+
+SUBPARSER = SUBPARSERS.add_parser(
+    _PARSER_NAME,
+    parents=[CRAWLER_PARSER],
+)
+
+GROUP = SUBPARSER.add_argument_group(
+    title=f"{_PARSER_NAME} crawler arguments",
+    description=f"arguments for the '{_PARSER_NAME}' crawler",
+)
+
+GROUP.add_argument(
+    "--ilias-url",
+    type=str,
+    metavar="ILIAS",
+    help="The base url of the ilias instance"
+)
+
+configure_common_group_args(GROUP)
+
+
+def load(args: argparse.Namespace, parser: configparser.ConfigParser) -> None:
+    log.explain(f"Creating config for command '{_PARSER_NAME}'")
+
+    parser["crawl:ilias"] = {}
+    section = parser["crawl:ilias"]
+    load_crawler(args, section)
+
+    section["type"] = _PARSER_NAME
+    section["ilias_url"] = str(args.ilias_url)
+
+    ilias_common_load(section, args, parser)
+
+
+SUBPARSER.set_defaults(command=load)

PFERD/cli/command_kit_ilias_web.py

@@ -1,120 +1,35 @@
 import argparse
 import configparser
-from pathlib import Path
 
-from ..crawl.ilias.file_templates import Links
+from .ilias_common import ilias_common_load, configure_common_group_args
+from .parser import CRAWLER_PARSER, SUBPARSERS, load_crawler
 from ..logging import log
-from .parser import (CRAWLER_PARSER, SUBPARSERS, BooleanOptionalAction, ParserLoadError, load_crawler,
+
-                     show_value_error)
+_PARSER_NAME = "kit-ilias-web"
 
 SUBPARSER = SUBPARSERS.add_parser(
-    "kit-ilias-web",
+    _PARSER_NAME,
     parents=[CRAWLER_PARSER],
 )
 
 GROUP = SUBPARSER.add_argument_group(
-    title="kit-ilias-web crawler arguments",
+    title=f"{_PARSER_NAME} crawler arguments",
-    description="arguments for the 'kit-ilias-web' crawler",
+    description=f"arguments for the '{_PARSER_NAME}' crawler",
-)
-GROUP.add_argument(
-    "target",
-    type=str,
-    metavar="TARGET",
-    help="course id, 'desktop', or ILIAS URL to crawl"
-)
-GROUP.add_argument(
-    "output",
-    type=Path,
-    metavar="OUTPUT",
-    help="output directory"
-)
-GROUP.add_argument(
-    "--username", "-u",
-    type=str,
-    metavar="USERNAME",
-    help="user name for authentication"
-)
-GROUP.add_argument(
-    "--keyring",
-    action=BooleanOptionalAction,
-    help="use the system keyring to store and retrieve passwords"
-)
-GROUP.add_argument(
-    "--credential-file",
-    type=Path,
-    metavar="PATH",
-    help="read username and password from a credential file"
-)
-GROUP.add_argument(
-    "--links",
-    type=show_value_error(Links.from_string),
-    metavar="OPTION",
-    help="how to represent external links"
-)
-GROUP.add_argument(
-    "--link-redirect-delay",
-    type=int,
-    metavar="SECONDS",
-    help="time before 'fancy' links redirect to to their target (-1 to disable)"
-)
-GROUP.add_argument(
-    "--videos",
-    action=BooleanOptionalAction,
-    help="crawl and download videos"
-)
-GROUP.add_argument(
-    "--forums",
-    action=BooleanOptionalAction,
-    help="crawl and download forum posts"
-)
-GROUP.add_argument(
-    "--http-timeout", "-t",
-    type=float,
-    metavar="SECONDS",
-    help="timeout for all HTTP requests"
 )
 
+configure_common_group_args(GROUP)
 
-def load(
+
-        args: argparse.Namespace,
+def load(args: argparse.Namespace, parser: configparser.ConfigParser) -> None:
-        parser: configparser.ConfigParser,
+    log.explain(f"Creating config for command '{_PARSER_NAME}'")
-) -> None:
-    log.explain("Creating config for command 'kit-ilias-web'")
 
     parser["crawl:ilias"] = {}
     section = parser["crawl:ilias"]
     load_crawler(args, section)
 
-    section["type"] = "kit-ilias-web"
+    section["type"] = _PARSER_NAME
-    section["target"] = str(args.target)
-    section["output_dir"] = str(args.output)
-    section["auth"] = "auth:ilias"
-    if args.links is not None:
-        section["links"] = str(args.links.value)
-    if args.link_redirect_delay is not None:
-        section["link_redirect_delay"] = str(args.link_redirect_delay)
-    if args.videos is not None:
-        section["videos"] = "yes" if args.videos else "no"
-    if args.forums is not None:
-        section["forums"] = "yes" if args.forums else "no"
-    if args.http_timeout is not None:
-        section["http_timeout"] = str(args.http_timeout)
 
-    parser["auth:ilias"] = {}
+    ilias_common_load(section, args, parser)
-    auth_section = parser["auth:ilias"]
-    if args.credential_file is not None:
-        if args.username is not None:
-            raise ParserLoadError("--credential-file and --username can't be used together")
-        if args.keyring:
-            raise ParserLoadError("--credential-file and --keyring can't be used together")
-        auth_section["type"] = "credential-file"
-        auth_section["path"] = str(args.credential_file)
-    elif args.keyring:
-        auth_section["type"] = "keyring"
-    else:
-        auth_section["type"] = "simple"
-    if args.username is not None:
-        auth_section["username"] = args.username
 
 
 SUBPARSER.set_defaults(command=load)

PFERD/cli/ilias_common.py

@@ -0,0 +1,102 @@
+import argparse
+import configparser
+from pathlib import Path
+
+from .parser import BooleanOptionalAction, ParserLoadError, show_value_error
+from ..crawl.ilias.file_templates import Links
+
+
+def configure_common_group_args(group: argparse._ArgumentGroup) -> None:
+    group.add_argument(
+        "target",
+        type=str,
+        metavar="TARGET",
+        help="course id, 'desktop', or ILIAS URL to crawl"
+    )
+    group.add_argument(
+        "output",
+        type=Path,
+        metavar="OUTPUT",
+        help="output directory"
+    )
+    group.add_argument(
+        "--username", "-u",
+        type=str,
+        metavar="USERNAME",
+        help="user name for authentication"
+    )
+    group.add_argument(
+        "--keyring",
+        action=BooleanOptionalAction,
+        help="use the system keyring to store and retrieve passwords"
+    )
+    group.add_argument(
+        "--credential-file",
+        type=Path,
+        metavar="PATH",
+        help="read username and password from a credential file"
+    )
+    group.add_argument(
+        "--links",
+        type=show_value_error(Links.from_string),
+        metavar="OPTION",
+        help="how to represent external links"
+    )
+    group.add_argument(
+        "--link-redirect-delay",
+        type=int,
+        metavar="SECONDS",
+        help="time before 'fancy' links redirect to to their target (-1 to disable)"
+    )
+    group.add_argument(
+        "--videos",
+        action=BooleanOptionalAction,
+        help="crawl and download videos"
+    )
+    group.add_argument(
+        "--forums",
+        action=BooleanOptionalAction,
+        help="crawl and download forum posts"
+    )
+    group.add_argument(
+        "--http-timeout", "-t",
+        type=float,
+        metavar="SECONDS",
+        help="timeout for all HTTP requests"
+    )
+
+
+def ilias_common_load(
+    section: configparser.SectionProxy,
+    args: argparse.Namespace,
+    parser: configparser.ConfigParser,
+) -> None:
+    section["target"] = str(args.target)
+    section["output_dir"] = str(args.output)
+    section["auth"] = "auth:ilias"
+    if args.links is not None:
+        section["links"] = str(args.links.value)
+    if args.link_redirect_delay is not None:
+        section["link_redirect_delay"] = str(args.link_redirect_delay)
+    if args.videos is not None:
+        section["videos"] = "yes" if args.videos else "no"
+    if args.forums is not None:
+        section["forums"] = "yes" if args.forums else "no"
+    if args.http_timeout is not None:
+        section["http_timeout"] = str(args.http_timeout)
+
+    parser["auth:ilias"] = {}
+    auth_section = parser["auth:ilias"]
+    if args.credential_file is not None:
+        if args.username is not None:
+            raise ParserLoadError("--credential-file and --username can't be used together")
+        if args.keyring:
+            raise ParserLoadError("--credential-file and --keyring can't be used together")
+        auth_section["type"] = "credential-file"
+        auth_section["path"] = str(args.credential_file)
+    elif args.keyring:
+        auth_section["type"] = "keyring"
+    else:
+        auth_section["type"] = "simple"
+    if args.username is not None:
+        auth_section["username"] = args.username