From 27e4abcfa32309eb3dd61ce20a4d65fa22a477a1 Mon Sep 17 00:00:00 2001 From: Joscha Date: Mon, 26 Apr 2021 23:46:44 +0200 Subject: [PATCH] Do project setup from scratch Following guidelines from the Python Packaging User Guide [1]. This commit intentionally breaks the .gitignore, project dependencies, GitHub Actions and other stuff. It also removes almost the entire README. The intention behind this is to get rid of all cruft that as accumulated over time and to have a fresh start. Only necessary things will be re-added as they're needed. From now on, I also plan on adding documentation for every feature at the same time that the feature is implemented. This is to ensure that the documentation does not become outdated. [1]: https://packaging.python.org/ --- .github/workflows/package.yml | 74 --------- .gitignore | 17 +- DEV.md | 37 +++++ README.md | 251 +---------------------------- example_config.py | 131 --------------- example_config_personal_desktop.py | 38 ----- pyproject.toml | 3 + requirements.txt | 4 - setup.cfg | 7 + setup.py | 17 -- 10 files changed, 56 insertions(+), 523 deletions(-) delete mode 100644 .github/workflows/package.yml create mode 100644 DEV.md delete mode 100644 example_config.py delete mode 100644 example_config_personal_desktop.py create mode 100644 pyproject.toml delete mode 100644 requirements.txt create mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index 615917b..0000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Package Application with Pyinstaller - -on: - push: - branches: - - "*" - tags: - - "v*" - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: "Install dependencies" - run: "pip install setuptools keyring pyinstaller rich requests beautifulsoup4 -f --upgrade" - - - name: "Install sync_url.py" - run: "pyinstaller sync_url.py -F" - - - name: "Move artifact" - run: "mv dist/sync_url* dist/sync_url-${{ matrix.os }}" - - - uses: actions/upload-artifact@v2 - with: - name: "Pferd Sync URL" - path: "dist/sync_url*" - - release: - name: Release - - needs: [build] - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: "Checkout" - uses: actions/checkout@v2 - - - name: "Download artifacts" - uses: actions/download-artifact@v2 - with: - name: "Pferd Sync URL" - - - name: "look at folder structure" - run: "ls -lah" - - - name: "Rename releases" - run: "mv sync_url-macos-latest pferd_sync_url_mac && mv sync_url-ubuntu-latest pferd_sync_url_linux && mv sync_url-windows-latest pferd_sync_url.exe" - - - name: "Create release" - uses: softprops/action-gh-release@v1 - - - name: "Upload release artifacts" - uses: softprops/action-gh-release@v1 - with: - body: "Download the correct sync_url for your platform and run it in the terminal or CMD. You might need to make it executable on Linux/Mac with `chmod +x `. Also please enclose the *url you pass to the program in double quotes* or your shell might silently screw it up!" - files: | - pferd_sync_url_mac - pferd_sync_url_linux - pferd_sync_url.exe diff --git a/.gitignore b/.gitignore index a5f87ba..bd8bab9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,2 @@ -__pycache__/ -.venv/ -venv/ -.idea/ -build/ -.mypy_cache/ -.tmp/ -.env -.vscode -ilias_cookies.txt -PFERD.egg-info/ - -# PyInstaller -sync_url.spec -dist/ +/.mypy_cache/ +/.venv/ diff --git a/DEV.md b/DEV.md new file mode 100644 index 0000000..a679b4a --- /dev/null +++ b/DEV.md @@ -0,0 +1,37 @@ +# PFERD Development Guide + +PFERD is packaged following the [Python Packaging User Guide][ppug] (in +particular [this][ppug-1] and [this][ppug-2] guide). + +[ppug]: "Python Packaging User Guide" +[ppug-1]: "Packaging Python Projects" +[ppug-2]: "Packaging and distributing projects" + +## Setting up a dev environment + +The use of [venv][venv] is recommended. To initially set up a development +environment, run these commands in the same directory as this file: + +``` +$ python -m venv .venv +$ . .venv/bin/activate +$ pip install --editable . +``` + +After this, you can use PFERD as if it was installed normally. Since PFERD was +installed with `--editable`, there is no need to re-run `pip install` when the +source code is changed. + +For more details, see [this part of the Python Tutorial][venv-tut] and +[this section on "development mode"][ppug-dev]. + +[venv]: "venv - Creation of virtual environments" +[venv-tut]: "12. Virtual Environments and Packages" +[ppug-dev]: "Working in “development mode”" + +## Contributing + +When submitting a PR that adds, changes or modifies a feature, please ensure +that the corresponding documentation is updated. + +In your first PR, please add your name to the `LICENSE` file. diff --git a/README.md b/README.md index 178fbac..5b74de5 100644 --- a/README.md +++ b/README.md @@ -2,254 +2,17 @@ **P**rogramm zum **F**lotten, **E**infachen **R**unterladen von **D**ateien -- [Quickstart with `sync_url`](#quickstart-with-sync_url) -- [Installation](#installation) - - [Upgrading from 2.0.0 to 2.1.0+](#upgrading-from-200-to-210) -- [Example setup](#example-setup) -- [Usage](#usage) - - [General concepts](#general-concepts) - - [Constructing transforms](#constructing-transforms) - - [Transform creators](#transform-creators) - - [Transform combinators](#transform-combinators) - - [A short, but commented example](#a-short-but-commented-example) +Other resources: -## Quickstart with `sync_url` +- [Development Guide](DEV.md) -The `sync_url` program allows you to just synchronize a given ILIAS URL (of a -course, a folder, your personal desktop, etc.) without any extra configuration -or setting up. Download the program, open ILIAS, copy the URL from the address -bar and pass it to sync_url. +## Installation with pip -It bundles everything it needs in one executable and is easy to -use, but doesn't expose all the configuration options and tweaks a full install -does. +Ensure you have at least Python 3.8 installed. Run the following command to +install PFERD or upgrade it to the latest version: -1. Download the `sync_url` binary from the [latest release](https://github.com/Garmelon/PFERD/releases/latest). -2. Recognize that you most likely need to enclose the URL in `""` quotes to prevent your shell from interpreting `&` and other symbols -3. Run the binary in your terminal (`./sync_url` or `sync_url.exe` in the CMD) to see the help and use it. I'd recommend using the `--cookies` option. - If you are on **Linux/Mac**, you need to *make the file executable* using `chmod +x `. - If you are on **Mac**, you need to allow this unverified program to run (see e.g. [here](https://www.switchingtomac.com/tutorials/osx/how-to-run-unverified-apps-on-macos/)) - -## Installation - -Ensure that you have at least Python 3.8 installed. - -To install PFERD or update your installation to the latest version, run this -wherever you want to install or have already installed PFERD: ``` -$ pip install git+https://github.com/Garmelon/PFERD@v2.6.1 +$ pip install --upgrade git+https://github.com/Garmelon/PFERD@latest ``` -The use of [venv] is recommended. - -[venv]: https://docs.python.org/3/library/venv.html - -### Upgrading from 2.0.0 to 2.1.0+ - -- The `IliasDirectoryType` type was renamed to `IliasElementType` and is now far more detailed. - The new values are: `REGULAR_FOLDER`, `VIDEO_FOLDER`, `EXERCISE_FOLDER`, `REGULAR_FILE`, `VIDEO_FILE`, `FORUM`, `EXTERNAL_LINK`. -- Forums and external links are skipped automatically if you use the `kit_ilias` helper. - -## Example setup - -In this example, `python3` refers to at least Python 3.8. - -A full example setup and initial use could look like: -``` -$ mkdir Vorlesungen -$ cd Vorlesungen -$ python3 -m venv .venv -$ source .venv/bin/activate -$ pip install git+https://github.com/Garmelon/PFERD@v2.6.1 -$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.6.1/example_config.py -$ python3 example_config.py -$ deactivate -``` - -Subsequent runs of the program might look like: -``` -$ cd Vorlesungen -$ source .venv/bin/activate -$ python3 example_config.py -$ deactivate -``` - -If you just want to get started and crawl *your entire ILIAS Desktop* instead -of a given set of courses, please replace `example_config.py` with -`example_config_personal_desktop.py` in all of the instructions below (`curl` call and -`python3` run command). - -## Usage - -### General concepts - -A PFERD config is a normal python file that starts multiple *synchronizers* -which do all the heavy lifting. While you can create and wire them up manually, -you are encouraged to use the helper methods provided in `PFERD.Pferd`. - -The synchronizers take some input arguments specific to their service and a -*transform*. The transform receives the computed path of an element in ILIAS and -can return either an output path (so you can rename files or move them around as -you wish) or `None` if you do not want to save the given file. - -Additionally the ILIAS synchronizer allows you to define a *crawl filter*. This -filter also receives the computed path as the input, but is only called for -*directories*. If you return `True`, the directory will be crawled and -searched. If you return `False` the directory will be ignored and nothing in it -will be passed to the transform. - -### Constructing transforms - -While transforms are just normal python functions, writing them by hand can -quickly become tedious. In order to help you with writing your own transforms -and filters, PFERD defines a few useful transform creators and combinators in -the `PFERD.transform` module: - -#### Transform creators - -These methods let you create a few basic transform building blocks: - -- **`glob(glob)`** - Creates a transform that returns the unchanged path if the glob matches the path and `None` otherwise. - See also [Path.match]. - Example: `glob("Übung/*.pdf")` -- **`predicate(pred)`** - Creates a transform that returns the unchanged path if `pred(path)` returns a truthy value. - Returns `None` otherwise. - Example: `predicate(lambda path: len(path.parts) == 3)` -- **`move_dir(source, target)`** - Creates a transform that moves all files from the `source` to the `target` directory. - Example: `move_dir("Übung/", "Blätter/")` -- **`move(source, target)`** - Creates a transform that moves the `source` file to `target`. - Example: `move("Vorlesung/VL02_Automten.pdf", "Vorlesung/VL02_Automaten.pdf")` -- **`rename(source, target)`** - Creates a transform that renames all files named `source` to `target`. - This transform works on the file names, not paths, and thus works no matter where the file is located. - Example: `rename("VL02_Automten.pdf", "VL02_Automaten.pdf")` -- **`re_move(regex, target)`** - Creates a transform that moves all files matching `regex` to `target`. - The transform `str.format` on the `target` string with the contents of the capturing groups before returning it. - The capturing groups can be accessed via their index. - See also [Match.group]. - Example: `re_move(r"Übung/Blatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf")` -- **`re_rename(regex, target)`** - Creates a transform that renames all files matching `regex` to `target`. - This transform works on the file names, not paths, and thus works no matter where the file is located. - Example: `re_rename(r"VL(\d+)(.*)\.pdf", "Vorlesung_Nr_{1}__{2}.pdf")` - -All movement or rename transforms above return `None` if a file doesn't match -their movement or renaming criteria. This enables them to be used as building -blocks to build up more complex transforms. - -In addition, `PFERD.transform` also defines the `keep` transform which returns its input path unchanged. -This behaviour can be very useful when creating more complex transforms. -See below for example usage. - -[Path.match]: https://docs.python.org/3/library/pathlib.html#pathlib.Path.match -[Match.group]: https://docs.python.org/3/library/re.html#re.Match.group - -#### Transform combinators - -These methods let you combine transforms into more complex transforms: - -- **`optionally(transform)`** - Wraps a given transform and returns its result if it is not `None`. - Otherwise returns the input path unchanged. - See below for example usage. -* **`do(transforms)`** - Accepts a series of transforms and applies them in the given order to the result of the previous one. - If any transform returns `None`, `do` short-circuits and also returns `None`. - This can be used to perform multiple renames in a row: - ```py - do( - # Move them - move_dir("Vorlesungsmaterial/Vorlesungsvideos/", "Vorlesung/Videos/"), - # Fix extensions (if they have any) - optionally(re_rename("(.*).m4v.mp4", "{1}.mp4")), - # Remove the 'dbs' prefix (if they have any) - optionally(re_rename("(?i)dbs-(.+)", "{1}")), - ) - ``` -- **`attempt(transforms)`** - Applies the passed transforms in the given order until it finds one that does not return `None`. - If it does not find any, it returns `None`. - This can be used to give a list of possible transformations and automatically pick the first one that fits: - ```py - attempt( - # Move all videos. If a video is passed in, this `re_move` will succeed - # and attempt short-circuits with the result. - re_move(r"Vorlesungsmaterial/.*/(.+?)\.mp4", "Vorlesung/Videos/{1}.mp4"), - # Move the whole folder to a nicer name - now without any mp4! - move_dir("Vorlesungsmaterial/", "Vorlesung/"), - # If we got another file, keep it. - keep, - ) - ``` - -All of these combinators are used in the provided example configs, if you want -to see some more real-life usages. - -### A short, but commented example - -```py -from pathlib import Path, PurePath -from PFERD import Pferd -from PFERD.ilias import IliasElementType -from PFERD.transform import * - -# This filter will later be used by the ILIAS crawler to decide whether it -# should crawl a directory (or directory-like structure). -def filter_course(path: PurePath, type: IliasElementType) -> bool: - # Note that glob returns a Transform, which is a function from PurePath -> - # Optional[PurePath]. Because of this, we need to apply the result of - # 'glob' to our input path. The returned value will be truthy (a Path) if - # the transform succeeded, or `None` if it failed. - - # We need to crawl the 'Tutorien' folder as it contains one that we want. - if glob("Tutorien/")(path): - return True - # If we found 'Tutorium 10', keep it! - if glob("Tutorien/Tutorium 10")(path): - return True - # Discard all other folders inside 'Tutorien' - if glob("Tutorien/*")(path): - return False - - # All other dirs (including subdirs of 'Tutorium 10') should be searched :) - return True - - -# This transform will later be used to rename a few files. It can also be used -# to ignore some files. -transform_course = attempt( - # We don't care about the other tuts and would instead prefer a cleaner - # directory structure. - move_dir("Tutorien/Tutorium 10/", "Tutorium/"), - # We don't want to modify any other files, so we're going to keep them - # exactly as they are. - keep -) - -# Enable and configure the text output. Needs to be called before calling any -# other PFERD methods. -Pferd.enable_logging() -# Create a Pferd instance rooted in the same directory as the script file. This -# is not a test run, so files will be downloaded (default, can be omitted). -pferd = Pferd(Path(__file__).parent, test_run=False) - -# Use the ilias_kit helper to synchronize an ILIAS course -pferd.ilias_kit( - # The directory that all of the downloaded files should be placed in - "My_cool_course/", - # The course ID (found in the URL when on the course page in ILIAS) - "course id", - # A path to a cookie jar. If you synchronize multiple ILIAS courses, - # setting this to a common value requires you to only log in once. - cookies=Path("ilias_cookies.txt"), - # A transform can rename, move or filter out certain files - transform=transform_course, - # A crawl filter limits what paths the cralwer searches - dir_filter=filter_course, -) -``` +The use of [venv](https://docs.python.org/3/library/venv.html) is recommended. diff --git a/example_config.py b/example_config.py deleted file mode 100644 index bffecfb..0000000 --- a/example_config.py +++ /dev/null @@ -1,131 +0,0 @@ -import argparse -from pathlib import Path, PurePath - -from PFERD import Pferd -from PFERD.ilias import IliasElementType -from PFERD.transform import (attempt, do, glob, keep, move, move_dir, - optionally, re_move, re_rename) - -tf_ss_2020_numerik = attempt( - re_move(r"Übungsblätter/(\d+)\. Übungsblatt/.*", "Blätter/Blatt_{1:0>2}.pdf"), - keep, -) - - -tf_ss_2020_db = attempt( - move_dir("Begrüßungsvideo/", "Vorlesung/Videos/"), - do( - move_dir("Vorlesungsmaterial/Vorlesungsvideos/", "Vorlesung/Videos/"), - optionally(re_rename("(.*).m4v.mp4", "{1}.mp4")), - optionally(re_rename("(?i)dbs-(.+)", "{1}")), - ), - move_dir("Vorlesungsmaterial/", "Vorlesung/"), - keep, -) - - -tf_ss_2020_rechnernetze = attempt( - re_move(r"Vorlesungsmaterial/.*/(.+?)\.mp4", "Vorlesung/Videos/{1}.mp4"), - move_dir("Vorlesungsmaterial/", "Vorlesung/"), - keep, -) - - -tf_ss_2020_sicherheit = attempt( - move_dir("Vorlesungsvideos/", "Vorlesung/Videos/"), - move_dir("Übungsvideos/", "Übung/Videos/"), - re_move(r"VL(.*)\.pdf", "Vorlesung/{1}.pdf"), - re_move(r"Übungsblatt (\d+)\.pdf", "Blätter/Blatt_{1:0>2}.pdf"), - move("Chiffrat.txt", "Blätter/Blatt_01_Chiffrat.txt"), - keep, -) - - -tf_ss_2020_pg = attempt( - move_dir("Vorlesungsaufzeichnungen/", "Vorlesung/Videos/"), - move_dir("Vorlesungsmaterial/", "Vorlesung/"), - re_move(r"Übungen/uebungsblatt(\d+).pdf", "Blätter/Blatt_{1:0>2}.pdf"), - keep, -) - - -def df_ss_2020_or1(path: PurePath, _type: IliasElementType) -> bool: - if glob("Tutorien/")(path): - return True - if glob("Tutorien/Tutorium 10, dienstags 15:45 Uhr/")(path): - return True - if glob("Tutorien/*")(path): - return False - return True - - -tf_ss_2020_or1 = attempt( - move_dir("Vorlesung/Unbeschriebene Folien/", "Vorlesung/Folien/"), - move_dir("Video zur Organisation/", "Vorlesung/Videos/"), - keep, -) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--test-run", action="store_true") - parser.add_argument("synchronizers", nargs="*") - args = parser.parse_args() - - pferd = Pferd(Path(__file__).parent, test_run=args.test_run) - pferd.enable_logging() - - if not args.synchronizers or "numerik" in args.synchronizers: - pferd.ilias_kit( - target="Numerik", - course_id="1083036", - transform=tf_ss_2020_numerik, - cookies="ilias_cookies.txt", - ) - - if not args.synchronizers or "db" in args.synchronizers: - pferd.ilias_kit( - target="DB", - course_id="1101554", - transform=tf_ss_2020_db, - cookies="ilias_cookies.txt", - ) - - if not args.synchronizers or "rechnernetze" in args.synchronizers: - pferd.ilias_kit( - target="Rechnernetze", - course_id="1099996", - transform=tf_ss_2020_rechnernetze, - cookies="ilias_cookies.txt", - ) - - if not args.synchronizers or "sicherheit" in args.synchronizers: - pferd.ilias_kit( - target="Sicherheit", - course_id="1101980", - transform=tf_ss_2020_sicherheit, - cookies="ilias_cookies.txt", - ) - - if not args.synchronizers or "pg" in args.synchronizers: - pferd.ilias_kit( - target="PG", - course_id="1106095", - transform=tf_ss_2020_pg, - cookies="ilias_cookies.txt", - ) - - if not args.synchronizers or "or1" in args.synchronizers: - pferd.ilias_kit( - target="OR1", - course_id="1105941", - dir_filter=df_ss_2020_or1, - transform=tf_ss_2020_or1, - cookies="ilias_cookies.txt", - ) - - # Prints a summary listing all new, modified or deleted files - pferd.print_summary() - -if __name__ == "__main__": - main() diff --git a/example_config_personal_desktop.py b/example_config_personal_desktop.py deleted file mode 100644 index 8d481b4..0000000 --- a/example_config_personal_desktop.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -This is a small config that just crawls the ILIAS Personal Desktop. -It does not filter or rename anything, it just gobbles up everything it can find. - -Note that this still includes a test-run switch, so you can see what it *would* download. -You can enable that with the "--test-run" command line switch, -i. e. "python3 example_config_minimal.py --test-run". -""" - -import argparse -from pathlib import Path - -from PFERD import Pferd - - -def main() -> None: - # Parse command line arguments - parser = argparse.ArgumentParser() - parser.add_argument("--test-run", action="store_true") - args = parser.parse_args() - - # Create the Pferd helper instance - pferd = Pferd(Path(__file__).parent, test_run=args.test_run) - pferd.enable_logging() - - # Synchronize the personal desktop into the "ILIAS" directory. - # It saves the cookies, so you only need to log in again when the ILIAS cookies expire. - pferd.ilias_kit_personal_desktop( - "ILIAS", - cookies="ilias_cookies.txt", - ) - - # Prints a summary listing all new, modified or deleted files - pferd.print_summary() - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2d852e1..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -requests>=2.21.0 -beautifulsoup4>=4.7.1 -rich>=2.1.0 -keyring>=21.5.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6d01c03 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,7 @@ +[metadata] +name = PFERD +version = 3.0.0 + +[options] +packages = PFERD +python_requires = >=3.8 diff --git a/setup.py b/setup.py deleted file mode 100644 index a4dfab3..0000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="PFERD", - version="2.6.1", - packages=find_packages(), - install_requires=[ - "requests>=2.21.0", - "beautifulsoup4>=4.7.1", - "rich>=2.1.0", - "keyring>=21.5.0" - ], -) - -# When updating the version, also: -# - update the README.md installation instructions -# - set a tag on the update commit