Compare commits

..

4 Commits

Author SHA1 Message Date
0b606f02fa Bump version 2021-01-17 10:33:10 +01:00
fb78a6e98e Retry ILIAS downloads a few times and only fail that file 2021-01-06 13:08:10 +01:00
5de68a0400 Bump version 2020-12-30 17:20:30 +01:00
f0562049b6 Remove Python 3.9 method in crawler 2020-12-30 17:18:04 +01:00
4 changed files with 24 additions and 8 deletions

View File

@ -281,7 +281,10 @@ class IliasCrawler:
result += [IliasCrawlerEntry(element_path, abs_url, element_type, None)]
continue
rest_of_name = meeting_name.removeprefix(date_portion_str)
rest_of_name = meeting_name
if rest_of_name.startswith(date_portion_str):
rest_of_name = rest_of_name[len(date_portion_str):]
new_name = datetime.datetime.strftime(date_portion, "%Y-%m-%d, %H:%M") \
+ rest_of_name
new_path = Path(folder_path, _sanitize_path_name(new_name))

View File

@ -122,9 +122,22 @@ class IliasDownloader:
tmp_file = self._tmp_dir.new_path()
while not self._try_download(info, tmp_file):
LOGGER.info("Retrying download: %r", info)
download_successful = False
for _ in range(0, 3):
try:
if not self._try_download(info, tmp_file):
LOGGER.info("Re-Authenticating due to download failure: %r", info)
self._authenticator.authenticate(self._session)
else:
download_successful = True
break
except IOError as e:
PRETTY.warning(f"I/O Error when downloading ({e}). Retrying...",)
LOGGER.info("Retrying download for %s", info.path)
if not download_successful:
PRETTY.error(f"Download of file {info.path} failed too often! Skipping it...")
return
dst_path = self._organizer.accept_file(tmp_file, info.path)
if dst_path and info.modification_date:

View File

@ -37,7 +37,7 @@ Ensure that you have at least Python 3.8 installed.
To install PFERD or update your installation to the latest version, run this
wherever you want to install or have already installed PFERD:
```
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.3
```
The use of [venv] is recommended.
@ -60,8 +60,8 @@ $ mkdir Vorlesungen
$ cd Vorlesungen
$ python3 -m venv .venv
$ .venv/bin/activate
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.1
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.1/example_config.py
$ pip install git+https://github.com/Garmelon/PFERD@v2.5.3
$ curl -O https://raw.githubusercontent.com/Garmelon/PFERD/v2.5.3/example_config.py
$ python3 example_config.py
$ deactivate
```

View File

@ -2,7 +2,7 @@ from setuptools import find_packages, setup
setup(
name="PFERD",
version="2.5.1",
version="2.5.3",
packages=find_packages(),
install_requires=[
"requests>=2.21.0",