From 3387bc5f20b6fdfef72b2620655547d1a2901e8b Mon Sep 17 00:00:00 2001 From: Christophe Date: Mon, 28 Sep 2020 17:49:36 +0200 Subject: [PATCH 1/9] Add simple course-download-by-url script --- sync_url.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 sync_url.py diff --git a/sync_url.py b/sync_url.py new file mode 100755 index 0000000..b2d3aca --- /dev/null +++ b/sync_url.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import argparse +from pathlib import Path +from urllib.parse import urlparse, parse_qs + +from PFERD import Pferd +from PFERD.cookie_jar import CookieJar +from PFERD.utils import to_path +from PFERD.ilias.authenticators import KitShibbolethAuthenticator +from PFERD.ilias.crawler import IliasCrawler + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--test-run", action="store_true") + parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in") + parser.add_argument('url', help="URL to the course page") + parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") + args = parser.parse_args() + + pferd = Pferd(Path(__file__).parent, test_run=args.test_run) + pferd.enable_logging() + + # parse provided course URL + url = urlparse(args.url) + query = parse_qs(url.query) + id = int(query['ref_id'][0]) + + if args.folder is None: + # fetch course name from ilias + cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) + session = cookie_jar.create_session() + authenticator = KitShibbolethAuthenticator() + crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True) + + cookie_jar.load_cookies() + folder = crawler.find_course_name(id) + cookie_jar.save_cookies() + else: + folder = args.folder + + # fetch + pferd.ilias_kit(target=folder, course_id=str(id), cookies=args.cookies) + +if __name__ == "__main__": + main() From be65051f9de3369a0d198a73008597f564b4138c Mon Sep 17 00:00:00 2001 From: Christophe Date: Mon, 28 Sep 2020 18:14:20 +0200 Subject: [PATCH 2/9] Support downloading folders in get-by-url script --- sync_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync_url.py b/sync_url.py index b2d3aca..ef4480a 100755 --- a/sync_url.py +++ b/sync_url.py @@ -34,7 +34,7 @@ def main() -> None: crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True) cookie_jar.load_cookies() - folder = crawler.find_course_name(id) + folder = crawler.find_element_name(args.url) cookie_jar.save_cookies() else: folder = args.folder From 74ea03945876c94c260b590e6140a7ee50630477 Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Mon, 28 Sep 2020 19:42:59 +0200 Subject: [PATCH 3/9] Fix a few lint errors and pferd quirks in sync_url --- sync_url.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sync_url.py b/sync_url.py index ef4480a..d486ce1 100755 --- a/sync_url.py +++ b/sync_url.py @@ -1,14 +1,19 @@ #!/usr/bin/env python +""" +A simple script to download a course by name from ILIAS. +""" + import argparse from pathlib import Path -from urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse from PFERD import Pferd from PFERD.cookie_jar import CookieJar -from PFERD.utils import to_path from PFERD.ilias.authenticators import KitShibbolethAuthenticator from PFERD.ilias.crawler import IliasCrawler +from PFERD.utils import to_path + def main() -> None: parser = argparse.ArgumentParser() @@ -18,29 +23,36 @@ def main() -> None: parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") args = parser.parse_args() - pferd = Pferd(Path(__file__).parent, test_run=args.test_run) - pferd.enable_logging() - # parse provided course URL url = urlparse(args.url) query = parse_qs(url.query) - id = int(query['ref_id'][0]) + course_id = query['ref_id'][0] - if args.folder is None: + if args.folder is not None: + folder = args.folder + # Initialize pferd at the *parent of the passed folder* + # This is needed so Pferd's internal protections against escaping the working directory + # do not trigger (e.g. if somebody names a file in ILIAS '../../bad thing.txt') + pferd = Pferd(Path(Path(__file__).parent, folder).parent, test_run=args.test_run) + else: # fetch course name from ilias cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) session = cookie_jar.create_session() authenticator = KitShibbolethAuthenticator() - crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, authenticator, lambda x, y: True) + crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, + authenticator, lambda x, y: True) cookie_jar.load_cookies() folder = crawler.find_element_name(args.url) cookie_jar.save_cookies() - else: - folder = args.folder + # Initialize pferd at the location of the script + pferd = Pferd(Path(__file__).parent, test_run=args.test_run) + + pferd.enable_logging() # fetch - pferd.ilias_kit(target=folder, course_id=str(id), cookies=args.cookies) + pferd.ilias_kit(target=folder, course_id=course_id, cookies=args.cookies) + if __name__ == "__main__": main() From 51a713fa04c2b1fe16a2ccdc0267147c906b65ee Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Mon, 28 Sep 2020 20:00:01 +0200 Subject: [PATCH 4/9] Allow crawling courses or folders with sync_url Video folders do not work, if they are passed directly. Their containing folder must be specified instead. --- PFERD/ilias/crawler.py | 10 +++++++ PFERD/pferd.py | 64 ++++++++++++++++++++++++++++++++++++++++++ sync_url.py | 20 ++++++------- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/PFERD/ilias/crawler.py b/PFERD/ilias/crawler.py index b6c65b2..a10db3d 100644 --- a/PFERD/ilias/crawler.py +++ b/PFERD/ilias/crawler.py @@ -116,6 +116,16 @@ class IliasCrawler: return urlunsplit((scheme, netloc, path, new_query_string, fragment)) + def recursive_crawl_url(self, url: str) -> List[IliasDownloadInfo]: + """ + Crawls a given url *and all reachable elements in it*. + + Args: + url {str} -- the *full* url to crawl + """ + start_entries: List[IliasCrawlerEntry] = self._crawl_folder(Path(""), url) + return self._iterate_entries_to_download_infos(start_entries) + def crawl_course(self, course_id: str) -> List[IliasDownloadInfo]: """ Starts the crawl process for a course, yielding a list of elements to (potentially) diff --git a/PFERD/pferd.py b/PFERD/pferd.py index 0b25151..042dd93 100644 --- a/PFERD/pferd.py +++ b/PFERD/pferd.py @@ -230,6 +230,70 @@ class Pferd(Location): return organizer + @swallow_and_print_errors + def ilias_kit_folder( + self, + target: PathLike, + full_url: str, + dir_filter: IliasDirectoryFilter = lambda x, y: True, + transform: Transform = lambda x: x, + cookies: Optional[PathLike] = None, + username: Optional[str] = None, + password: Optional[str] = None, + download_strategy: IliasDownloadStrategy = download_modified_or_new, + clean: bool = True, + timeout: int = 5, + ) -> Organizer: + """ + Synchronizes a folder with a given folder on the ILIAS instance of the KIT. + + Arguments: + target {Path} -- the target path to write the data to + full_url {str} -- the full url of the folder/videos/course to crawl + + Keyword Arguments: + dir_filter {IliasDirectoryFilter} -- A filter for directories. Will be applied on the + crawler level, these directories and all of their content is skipped. + (default: {lambdax:True}) + transform {Transform} -- A transformation function for the output paths. Return None + to ignore a file. (default: {lambdax:x}) + cookies {Optional[Path]} -- The path to store and load cookies from. + (default: {None}) + username {Optional[str]} -- The SCC username. If none is given, it will prompt + the user. (default: {None}) + password {Optional[str]} -- The SCC password. If none is given, it will prompt + the user. (default: {None}) + download_strategy {DownloadStrategy} -- A function to determine which files need to + be downloaded. Can save bandwidth and reduce the number of requests. + (default: {download_modified_or_new}) + clean {bool} -- Whether to clean up when the method finishes. + timeout {int} -- The download timeout for opencast videos. Sadly needed due to a + requests bug. + """ + # This authenticator only works with the KIT ilias instance. + authenticator = KitShibbolethAuthenticator(username=username, password=password) + PRETTY.starting_synchronizer(target, "ILIAS", "An ILIAS element by url") + + if not full_url.startswith("https://ilias.studium.kit.edu"): + raise FatalException("Not a valid KIT ILIAS URL") + + organizer = self._ilias( + target=target, + base_url="https://ilias.studium.kit.edu/", + crawl_function=lambda crawler: crawler.recursive_crawl_url(full_url), + authenticator=authenticator, + cookies=cookies, + dir_filter=dir_filter, + transform=transform, + download_strategy=download_strategy, + clean=clean, + timeout=timeout + ) + + self._download_summary.merge(organizer.download_summary) + + return organizer + @swallow_and_print_errors def diva_kit( self, diff --git a/sync_url.py b/sync_url.py index d486ce1..64c742b 100755 --- a/sync_url.py +++ b/sync_url.py @@ -23,10 +23,15 @@ def main() -> None: parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") args = parser.parse_args() - # parse provided course URL url = urlparse(args.url) - query = parse_qs(url.query) - course_id = query['ref_id'][0] + + cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) + session = cookie_jar.create_session() + authenticator = KitShibbolethAuthenticator() + crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, + authenticator, lambda x, y: True) + + cookie_jar.load_cookies() if args.folder is not None: folder = args.folder @@ -36,13 +41,6 @@ def main() -> None: pferd = Pferd(Path(Path(__file__).parent, folder).parent, test_run=args.test_run) else: # fetch course name from ilias - cookie_jar = CookieJar(to_path(args.cookies) if args.cookies else None) - session = cookie_jar.create_session() - authenticator = KitShibbolethAuthenticator() - crawler = IliasCrawler(url.scheme + '://' + url.netloc, session, - authenticator, lambda x, y: True) - - cookie_jar.load_cookies() folder = crawler.find_element_name(args.url) cookie_jar.save_cookies() @@ -51,7 +49,7 @@ def main() -> None: pferd.enable_logging() # fetch - pferd.ilias_kit(target=folder, course_id=course_id, cookies=args.cookies) + pferd.ilias_kit_folder(target=folder, full_url=args.url, cookies=args.cookies) if __name__ == "__main__": From c1ccb6c53ee27af2587bf4918e764ffe268bbb58 Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Tue, 6 Oct 2020 10:39:34 +0200 Subject: [PATCH 5/9] Allow crawling videos with sync_url --- PFERD/ilias/crawler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/PFERD/ilias/crawler.py b/PFERD/ilias/crawler.py index a10db3d..ca00771 100644 --- a/PFERD/ilias/crawler.py +++ b/PFERD/ilias/crawler.py @@ -245,6 +245,12 @@ class IliasCrawler: """ soup = self._get_page(url, {}) + if soup.find(id="headerimage"): + element: bs4.Tag = soup.find(id="headerimage") + if "opencast" in element.attrs["src"].lower(): + PRETTY.warning(f"Switched to crawling a video at {folder_path}") + return self._crawl_video_directory(folder_path, url) + result: List[IliasCrawlerEntry] = [] # Fetch all links and throw them to the general interpreter From b3ad9783c4cf5fc28731889e95b74b7a89da4077 Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Tue, 6 Oct 2020 11:43:20 +0200 Subject: [PATCH 6/9] Ignore pyinstaller files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 0d18aa7..fbb852b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ build/ .env .vscode ilias_cookies.txt + +# PyInstaller +sync_url.spec +dist/ From a519cbe05dc667eeb272a745bc1ce1ee2776ae3c Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Tue, 6 Oct 2020 12:03:16 +0200 Subject: [PATCH 7/9] Add sync_url workflow --- .github/workflows/package.yml | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 0000000..c7111b7 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,67 @@ +name: Package Application with Pyinstaller + +on: + push: + branches: + - "*" + tags: + - "v*" + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: "Install dependencies" + run: "pip install setuptools pyinstaller rich requests beautifulsoup4 -f --upgrade" + + - name: "Install sync_url.py" + run: "pyinstaller sync_url.py -F" + + - uses: actions/upload-artifact@v2 + with: + name: "Pferd Sync URL" + path: "dist/sync_url*" + + release: + name: Release + + needs: [build] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: "Checkout" + uses: actions/checkout@v2 + + - name: "Download artifacts" + uses: actions/download-artifact@v2 + with: + name: "Pferd Sync URL" + + - name: "look at folder structure" + run: "ls -lah" + + - name: "Create release" + uses: softprops/action-gh-release@v1 + + - name: "Upload release artifacts" + uses: softprops/action-gh-release@v1 + with: + body: "Download sync_url (or sync_url.exe on Windows) and run it in the terminal or CMD." + files: | + sync_url + sync_url.exe From 73c3eb098450736da10c45ccadec8f9ed3b4b4cd Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Tue, 6 Oct 2020 17:15:10 +0200 Subject: [PATCH 8/9] Add option to skip videos in sync_url --- PFERD/ilias/crawler.py | 3 +++ sync_url.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/PFERD/ilias/crawler.py b/PFERD/ilias/crawler.py index ca00771..7ce460e 100644 --- a/PFERD/ilias/crawler.py +++ b/PFERD/ilias/crawler.py @@ -249,6 +249,9 @@ class IliasCrawler: element: bs4.Tag = soup.find(id="headerimage") if "opencast" in element.attrs["src"].lower(): PRETTY.warning(f"Switched to crawling a video at {folder_path}") + if not self.dir_filter(folder_path, IliasElementType.VIDEO_FOLDER): + PRETTY.not_searching(folder_path, "user filter") + return [] return self._crawl_video_directory(folder_path, url) result: List[IliasCrawlerEntry] = [] diff --git a/sync_url.py b/sync_url.py index 64c742b..6da60dd 100755 --- a/sync_url.py +++ b/sync_url.py @@ -6,12 +6,12 @@ A simple script to download a course by name from ILIAS. import argparse from pathlib import Path -from urllib.parse import parse_qs, urlparse +from urllib.parse import urlparse from PFERD import Pferd from PFERD.cookie_jar import CookieJar -from PFERD.ilias.authenticators import KitShibbolethAuthenticator -from PFERD.ilias.crawler import IliasCrawler +from PFERD.ilias import (IliasCrawler, IliasElementType, + KitShibbolethAuthenticator) from PFERD.utils import to_path @@ -19,6 +19,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--test-run", action="store_true") parser.add_argument('-c', '--cookies', nargs='?', default=None, help="File to store cookies in") + parser.add_argument('-f', '--no-videos', nargs='?', default=None, help="Don't download videos") parser.add_argument('url', help="URL to the course page") parser.add_argument('folder', nargs='?', default=None, help="Folder to put stuff into") args = parser.parse_args() @@ -47,9 +48,19 @@ def main() -> None: # Initialize pferd at the location of the script pferd = Pferd(Path(__file__).parent, test_run=args.test_run) + def dir_filter(_: Path, element: IliasElementType) -> bool: + if args.no_videos: + return element not in [IliasElementType.VIDEO_FILE, IliasElementType.VIDEO_FOLDER] + return True + pferd.enable_logging() # fetch - pferd.ilias_kit_folder(target=folder, full_url=args.url, cookies=args.cookies) + pferd.ilias_kit_folder( + target=folder, + full_url=args.url, + cookies=args.cookies, + dir_filter=dir_filter + ) if __name__ == "__main__": From d73c778b0a0ba90ee42cf5eed7d4265694e2203a Mon Sep 17 00:00:00 2001 From: I-Al-Istannen Date: Tue, 6 Oct 2020 17:46:42 +0200 Subject: [PATCH 9/9] Add sync_url instructions to README --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 454fd90..24e73a6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ **P**rogramm zum **F**lotten, **E**infachen **R**unterladen von **D**ateien +- [Quickstart with `sync_url`](#quickstart-with-sync_url) - [Installation](#installation) - [Upgrading from 2.0.0 to 2.1.0+](#upgrading-from-200-to-210) - [Example setup](#example-setup) @@ -12,6 +13,20 @@ - [Transform combinators](#transform-combinators) - [A short, but commented example](#a-short-but-commented-example) +## Quickstart with `sync_url` + +The `sync_url` program allows you to just synchronize a given ILIAS URL (of a +course, a folder, your personal desktop, etc.) without any extra configuration +or setting up. Download the program, open ILIAS, copy the URL from the address +bar and pass it to sync_url. + +It bundles everything it needs in one executable and is easy to +use, but doesn't expose all the configuration options and tweaks a full install +does. + +1. Download the `sync_url` binary from the [latest release](https://github.com/Garmelon/PFERD/releases/latest). +2. Run the binary in your terminal (`./sync_url` or `sync_url.exe` in the CMD) to see the help and use it + ## Installation Ensure that you have at least Python 3.8 installed.