mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Create crawler base dir at start of crawl

Browse Source
This commit is contained in:
Joscha 2021-05-23 10:52:02 +02:00
parent 729ff0a4c7
commit ec3767c545
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
PFERD/crawler.py
View File

@ -269,6 +269,7 @@ class Crawler(ABC):
"""
with log.show_progress():
self._output_dir.prepare()
await self._run()
await self._cleanup()

11
PFERD/output_dir.py
View File

@ -142,6 +142,14 @@ class OutputDirectory:
self._report = Report()
def prepare(self) -> None:
log.explain_topic(f"Creating base directory at {str(self._root.absolute())!r}")
try:
self._root.mkdir(parents=True, exist_ok=True)
except OSError:
raise OutputDirError("Failed to create base directory")
def register_reserved(self, path: PurePath) -> None:
self._report.mark_reserved(path)
@ -385,9 +393,6 @@ class OutputDirectory:
self._report.add_file(info.path)
async def cleanup(self) -> None:
if not self._root.exists():
return
await self._cleanup_dir(self._root, PurePath(), delete_self=False)
async def _cleanup(self, path: Path, pure: PurePath) -> None: