mirrors/pferd
1
0
Fork 0
mirror of https://github.com/Garmelon/PFERD.git synced 2025-10-20 00:32:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Reformat and switch to ruff

Browse Source
This commit is contained in:
I-Al-Istannen
2025-10-19 15:19:43 +02:00
parent ee4625be78
commit 2cf0e060ed
31 changed files with 1507 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
PFERD/crawl/crawler.py
View File

@@ -132,8 +132,9 @@ class DownloadToken(ReusableAsyncContextManager[Tuple[ProgressBar, FileSink]]):
await self._stack.enter_async_context(self._limiter.limit_download())
sink = await self._stack.enter_async_context(self._fs_token)
# The "Downloaded ..." message is printed in the output dir, not here
bar = self._stack.enter_context(log.download_bar("[bold bright_cyan]", "Downloading",
fmt_path(self._path)))
bar = self._stack.enter_context(
log.download_bar("[bold bright_cyan]", "Downloading", fmt_path(self._path))
)
return bar, sink
@@ -216,10 +217,10 @@ class CrawlerSection(Section):
class Crawler(ABC):
def __init__(
self,
name: str,
section: CrawlerSection,
config: Config,
self,
name: str,
section: CrawlerSection,
config: Config,
) -> None:
"""
Initialize a crawler from its name and its section in the config file.
@@ -293,13 +294,13 @@ class Crawler(ABC):
return CrawlToken(self._limiter, path)
def should_try_download(
self,
path: PurePath,
*,
etag_differs: Optional[bool] = None,
mtime: Optional[datetime] = None,
redownload: Optional[Redownload] = None,
on_conflict: Optional[OnConflict] = None,
self,
path: PurePath,
*,
etag_differs: Optional[bool] = None,
mtime: Optional[datetime] = None,
redownload: Optional[Redownload] = None,
on_conflict: Optional[OnConflict] = None,
) -> bool:
log.explain_topic(f"Decision: Should Download {fmt_path(path)}")
@@ -308,11 +309,7 @@ class Crawler(ABC):
return False
should_download = self._output_dir.should_try_download(
path,
etag_differs=etag_differs,
mtime=mtime,
redownload=redownload,
on_conflict=on_conflict
path, etag_differs=etag_differs, mtime=mtime, redownload=redownload, on_conflict=on_conflict
)
if should_download:
log.explain("Answer: Yes")
@@ -322,13 +319,13 @@ class Crawler(ABC):
return False
async def download(
self,
path: PurePath,
*,
etag_differs: Optional[bool] = None,
mtime: Optional[datetime] = None,
redownload: Optional[Redownload] = None,
on_conflict: Optional[OnConflict] = None,
self,
path: PurePath,
*,
etag_differs: Optional[bool] = None,
mtime: Optional[datetime] = None,
redownload: Optional[Redownload] = None,
on_conflict: Optional[OnConflict] = None,
) -> Optional[DownloadToken]:
log.explain_topic(f"Decision: Download {fmt_path(path)}")
path = self._deduplicator.mark(path)
@@ -346,7 +343,7 @@ class Crawler(ABC):
etag_differs=etag_differs,
mtime=mtime,
redownload=redownload,
on_conflict=on_conflict
on_conflict=on_conflict,
)
if fs_token is None:
log.explain("Answer: No")