mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make file cleanup part of default crawler behaviour

Browse Source
This commit is contained in:
Joscha 2021-05-22 21:45:51 +02:00
parent ec95dda18f
commit e21795ee35
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
PFERD/crawler.py
View File

@ -244,8 +244,15 @@ class Crawler(ABC):
desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}" desc = f"[bold bright_cyan]Downloading[/] {escape(str(path))}"
return DownloadToken(self._limiter, fs_token, desc) return DownloadToken(self._limiter, fs_token, desc)
async def cleanup(self) -> None: async def _cleanup(self) -> None:
log.explain_topic("Decision: Clean up files?")
if self.error_free:
log.explain("No warnings or errors occurred during this run")
log.explain("Cleaning up files")
await self._output_dir.cleanup() await self._output_dir.cleanup()
else:
log.explain("Warnings or errors occurred during this run")
log.explain("Not cleaning up files")
async def run(self) -> None: async def run(self) -> None:
""" """
@ -255,6 +262,7 @@ class Crawler(ABC):
with log.show_progress(): with log.show_progress():
await self._run() await self._run()
await self._cleanup()
@abstractmethod @abstractmethod
async def _run(self) -> None: async def _run(self) -> None: