mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2023-12-21 10:23:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Load and store reports

Browse Source
This commit is contained in:
Joscha
2021-05-23 20:46:12 +02:00
parent 6e9f8fd391
commit 6ca0ecdf05
3 changed files with 110 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
PFERD/crawl/crawler.py
View File

@ -11,7 +11,7 @@ from ..config import Config, Section
from ..limiter import Limiter
from ..logging import ProgressBar, log
from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload
from ..report import MarkConflictError, MarkDuplicateError
from ..report import MarkConflictError, MarkDuplicateError, Report
from ..transformer import Transformer
from ..utils import ReusableAsyncContextManager, fmt_path
@ -229,6 +229,14 @@ class Crawler(ABC):
section.on_conflict(),
)
@property
def report(self) -> Report:
return self._output_dir.report
@property
def prev_report(self) -> Optional[Report]:
return self._output_dir.prev_report
@staticmethod
async def gather(awaitables: Sequence[Awaitable[Any]]) -> List[Any]:
"""
@ -298,8 +306,10 @@ class Crawler(ABC):
with log.show_progress():
self._output_dir.prepare()
self._output_dir.load_prev_report()
await self._run()
await self._cleanup()
self._output_dir.store_report()
@abstractmethod
async def _run(self) -> None: