mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Remove unnecessary files
This commit is contained in:
Joscha
parent
dfddc93039
commit
f1ba618378
|
|
@ -1,80 +0,0 @@
|
|||
# From https://charemza.name/blog/posts/python/asyncio/read-write-lock/
|
||||
# https://gist.github.com/michalc/ab9bd571cfab09216c0316f2302a76b0#file-asyncio_read_write_lock-py
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import contextlib
|
||||
|
||||
|
||||
class _ReadWaiter(asyncio.Future):
|
||||
pass
|
||||
|
||||
class _WriteWaiter(asyncio.Future):
|
||||
pass
|
||||
|
||||
class ReadWriteLock():
|
||||
|
||||
def __init__(self):
|
||||
self._waiters = collections.deque()
|
||||
self._reads_held = 0
|
||||
self._write_held = False
|
||||
|
||||
def _pop_queued_waiters(self, waiter_type):
|
||||
while True:
|
||||
correct_type = self._waiters and isinstance(self._waiters[0], waiter_type)
|
||||
cancelled = self._waiters and self._waiters[0].cancelled()
|
||||
|
||||
if correct_type or cancelled:
|
||||
waiter = self._waiters.popleft()
|
||||
|
||||
if correct_type and not cancelled:
|
||||
yield waiter
|
||||
|
||||
if not correct_type and not cancelled:
|
||||
break
|
||||
|
||||
def _resolve_queued_waiters(self):
|
||||
if not self._write_held:
|
||||
for waiter in self._pop_queued_waiters(_ReadWaiter):
|
||||
self._reads_held += 1
|
||||
waiter.set_result(None)
|
||||
|
||||
if not self._write_held and not self._reads_held:
|
||||
for waiter in self._pop_queued_waiters(_WriteWaiter):
|
||||
self._write_held = True
|
||||
waiter.set_result(None)
|
||||
break
|
||||
|
||||
def _on_read_release(self):
|
||||
self._reads_held -= 1
|
||||
|
||||
def _on_write_release(self):
|
||||
self._write_held = False
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _acquire(self, waiter_type, on_release):
|
||||
waiter = waiter_type()
|
||||
self._waiters.append(waiter)
|
||||
self._resolve_queued_waiters()
|
||||
|
||||
try:
|
||||
await waiter
|
||||
except asyncio.CancelledError:
|
||||
self._resolve_queued_waiters()
|
||||
raise
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
on_release()
|
||||
self._resolve_queued_waiters()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def read(self):
|
||||
async with self._acquire(_ReadWaiter, self._on_read_release):
|
||||
yield
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def write(self):
|
||||
async with self._acquire(_WriteWaiter, self._on_write_release):
|
||||
yield
|
||||
87
plan.md
87
plan.md
|
|
@ -1,87 +0,0 @@
|
|||
# Config
|
||||
|
||||
- python script as config
|
||||
- imports PFERD as library
|
||||
- operates relative to its own path
|
||||
|
||||
## Example folder structure
|
||||
|
||||
```
|
||||
.../
|
||||
Vorlesungen/
|
||||
PFERD/ as symlink, locally or in python's import path
|
||||
PFERDconf.py does things relative to its own location (hopefully)
|
||||
GBI/ folder to synchronize files into
|
||||
...
|
||||
Prog/ folder to synchronize files into
|
||||
...
|
||||
```
|
||||
|
||||
## Example config
|
||||
|
||||
```python
|
||||
import PFERD
|
||||
|
||||
def gbi_filter(ilias_path):
|
||||
... # rename and move files, or filter them
|
||||
return local_path # or None if file should be ignored
|
||||
|
||||
kit = PFERD.KIT()
|
||||
kit.synchronize("crs_855240", "GBI", filter=gbi_filter)
|
||||
kit.synchronize("crs_851237", "Prog") # default filter preserves paths
|
||||
```
|
||||
|
||||
# Structure
|
||||
|
||||
## Things that need to be done
|
||||
|
||||
- figure out where config file is located
|
||||
- get and store shibboleth session cookie
|
||||
- get and store ilias session cookie
|
||||
- download specific ilias urls (mostly goto.php, probably)
|
||||
- parse web page
|
||||
- determine if logging in is necessary
|
||||
- authenticate if necessary
|
||||
- don't re-login if shibboleth session cookie is still valid
|
||||
- find folders in current folder
|
||||
- find files to download in current folder
|
||||
- ignore LA1 test thingy
|
||||
- crawl folders and create directory-like structure/file paths
|
||||
- use filter function for paths
|
||||
- download files to local file paths
|
||||
- create folders as necessary
|
||||
- remember downloaded files
|
||||
- find files that were not previously downloaded
|
||||
- remove un-downloaded files
|
||||
- remove unnecessary folders (prompt user before deleting!)
|
||||
- logging
|
||||
- display crawl progress
|
||||
- display structure of found files using neat box drawing characters
|
||||
- display download progress
|
||||
|
||||
## How those things are usually done
|
||||
|
||||
Step 3. to 5. are run for each synchronize() call.
|
||||
|
||||
1. launch script
|
||||
- load cookie files
|
||||
2. authenticate
|
||||
(assuming enough time has passed for the session cookies to become invalid)
|
||||
- prompt user for username
|
||||
- prompt user for password (no echo)
|
||||
- somehow obtain valid session cookies
|
||||
3. crawl
|
||||
- start at the id specified in synchronize() args
|
||||
- search for folders and files to download
|
||||
- build directory structure
|
||||
4. download
|
||||
- run each path through filter function
|
||||
- if file was not filtered:
|
||||
- download file and save result to filtered path
|
||||
- use sync directory specified in synchronize() args
|
||||
- remember the filtered path for later
|
||||
5. cleanup
|
||||
- search sync directory for files
|
||||
- for each file not previously downloaded:
|
||||
- prompt user if they want to delete the file (default: No)
|
||||
- delete file if user answered Yes
|
||||
44
test.py
44
test.py
|
|
@ -1,44 +0,0 @@
|
|||
import PFERD
|
||||
import asyncio
|
||||
import logging
|
||||
import pathlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG, format=PFERD.LOG_FORMAT)
|
||||
#logging.basicConfig(level=logging.INFO, format=PFERD.LOG_FORMAT)
|
||||
|
||||
async def test_download():
|
||||
base_path = pathlib.Path(".")
|
||||
sync_path = pathlib.Path(base_path, "synctest")
|
||||
|
||||
orga = PFERD.Organizer(base_path, sync_path)
|
||||
auth = PFERD.ShibbolethAuthenticator(cookie_path="cookie_jar")
|
||||
#soup = await auth.get_webpage("885157")
|
||||
|
||||
orga.clean_temp_dir()
|
||||
|
||||
filename = orga.temp_file()
|
||||
await auth.download_file("file_886544_download", filename)
|
||||
orga.add_file(filename, pathlib.Path("test.pdf"))
|
||||
|
||||
filename = orga.temp_file()
|
||||
await auth.download_file("file_886544_download", filename)
|
||||
orga.add_file(filename, pathlib.Path("bla/test2.pdf"))
|
||||
|
||||
orga.clean_sync_dir()
|
||||
orga.clean_temp_dir()
|
||||
await auth.close()
|
||||
|
||||
def main():
|
||||
#print(f" os.getcwd(): {os.getcwd()}")
|
||||
#print(f" sys.argv[0]: {sys.argv[0]}")
|
||||
#print(f" both: {os.path.dirname(os.getcwd() + '/' + sys.argv[0])}")
|
||||
#print(f" __file__: {__file__}")
|
||||
#print(f"stackoverflow: {os.path.dirname(os.path.abspath(__file__))}")
|
||||
|
||||
#asyncio.run(test_download(), debug=True)
|
||||
asyncio.run(test_download())
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user