Create crawler base dir at start of crawl

This commit is contained in:
Joscha 2021-05-23 10:52:02 +02:00
parent 729ff0a4c7
commit ec3767c545
2 changed files with 9 additions and 3 deletions

View File

@ -269,6 +269,7 @@ class Crawler(ABC):
""" """
with log.show_progress(): with log.show_progress():
self._output_dir.prepare()
await self._run() await self._run()
await self._cleanup() await self._cleanup()

View File

@ -142,6 +142,14 @@ class OutputDirectory:
self._report = Report() self._report = Report()
def prepare(self) -> None:
log.explain_topic(f"Creating base directory at {str(self._root.absolute())!r}")
try:
self._root.mkdir(parents=True, exist_ok=True)
except OSError:
raise OutputDirError("Failed to create base directory")
def register_reserved(self, path: PurePath) -> None: def register_reserved(self, path: PurePath) -> None:
self._report.mark_reserved(path) self._report.mark_reserved(path)
@ -385,9 +393,6 @@ class OutputDirectory:
self._report.add_file(info.path) self._report.add_file(info.path)
async def cleanup(self) -> None: async def cleanup(self) -> None:
if not self._root.exists():
return
await self._cleanup_dir(self._root, PurePath(), delete_self=False) await self._cleanup_dir(self._root, PurePath(), delete_self=False)
async def _cleanup(self, path: Path, pure: PurePath) -> None: async def _cleanup(self, path: Path, pure: PurePath) -> None: