mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Allow crawling courses or folders with sync_url

Browse Source 
Video folders do not work, if they are passed directly. Their containing
folder must be specified instead.
This commit is contained in:
I-Al-Istannen
2020-09-28 20:00:01 +02:00
parent 74ea039458
commit 51a713fa04
3 changed files with 83 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
PFERD/ilias/crawler.py
View File

@ -116,6 +116,16 @@ class IliasCrawler:
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def recursive_crawl_url(self, url: str) -> List[IliasDownloadInfo]:
"""
Crawls a given url *and all reachable elements in it*.
Args:
url {str} -- the *full* url to crawl
"""
start_entries: List[IliasCrawlerEntry] = self._crawl_folder(Path(""), url)
return self._iterate_entries_to_download_infos(start_entries)
def crawl_course(self, course_id: str) -> List[IliasDownloadInfo]:
"""
Starts the crawl process for a course, yielding a list of elements to (potentially)