Use PurePath instead of Path

Path should only be used when we need to access the file system. For all other
purposes (mainly crawling), we use PurePath instead since the paths don't
correspond to paths in the local file system.
This commit is contained in:
Joscha 2021-04-29 16:52:00 +02:00
parent 0096d83387
commit f776186480
3 changed files with 21 additions and 20 deletions

View File

@ -1,7 +1,7 @@
import configparser import configparser
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from pathlib import Path from pathlib import PurePath
# TODO In Python 3.9 and above, AsyncContextManager is deprecated # TODO In Python 3.9 and above, AsyncContextManager is deprecated
from typing import AsyncContextManager, AsyncIterator, Optional from typing import AsyncContextManager, AsyncIterator, Optional
@ -38,7 +38,8 @@ class Crawler(ABC):
e.pretty_print() e.pretty_print()
raise CrawlerLoadException() raise CrawlerLoadException()
# output_dir = Path(section.get("output_dir", name)) # working_dir = Path(section.get("working_dir", ""))
# output_dir = working_dir / section.get("output_dir", name)
def print(self, text: str) -> None: def print(self, text: str) -> None:
""" """
@ -75,14 +76,14 @@ class Crawler(ABC):
with self._conductor.progress_bar(desc, total=total) as bar: with self._conductor.progress_bar(desc, total=total) as bar:
yield bar yield bar
def crawl_bar(self, path: Path) -> AsyncContextManager[ProgressBar]: def crawl_bar(self, path: PurePath) -> AsyncContextManager[ProgressBar]:
pathstr = escape(str(path)) pathstr = escape(str(path))
desc = f"[bold magenta]Crawling[/bold magenta] {pathstr}" desc = f"[bold magenta]Crawling[/bold magenta] {pathstr}"
return self.progress_bar(desc) return self.progress_bar(desc)
def download_bar( def download_bar(
self, self,
path: Path, path: PurePath,
size: int, size: int,
) -> AsyncContextManager[ProgressBar]: ) -> AsyncContextManager[ProgressBar]:
pathstr = escape(str(path)) pathstr = escape(str(path))

View File

@ -1,6 +1,6 @@
import asyncio import asyncio
import random import random
from pathlib import Path from pathlib import PurePath
from typing import Any from typing import Any
from rich.markup import escape from rich.markup import escape
@ -37,9 +37,9 @@ DUMMY_TREE = {
class DummyCrawler(Crawler): class DummyCrawler(Crawler):
async def crawl(self) -> None: async def crawl(self) -> None:
await self._crawl_entry(Path(), DUMMY_TREE) await self._crawl_entry(PurePath(), DUMMY_TREE)
async def _crawl_entry(self, path: Path, value: Any) -> None: async def _crawl_entry(self, path: PurePath, value: Any) -> None:
if value is True: if value is True:
async with self.exclusive_output(): async with self.exclusive_output():
await ainput(f"File {path}, please press enter: ") await ainput(f"File {path}, please press enter: ")

View File

@ -1,22 +1,22 @@
import re import re
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import PurePath
from typing import Dict, Optional, Union from typing import Dict, Optional, Union
class Rule(ABC): class Rule(ABC):
@abstractmethod @abstractmethod
def transform(self, path: Path) -> Optional[Path]: def transform(self, path: PurePath) -> Optional[PurePath]:
pass pass
class NormalRule(Rule): class NormalRule(Rule):
def __init__(self, left: Path, right: Path): def __init__(self, left: PurePath, right: PurePath):
self._left = left self._left = left
self._right = right self._right = right
def _match_prefix(self, path: Path) -> Optional[Path]: def _match_prefix(self, path: PurePath) -> Optional[PurePath]:
left_parts = list(reversed(self._left.parts)) left_parts = list(reversed(self._left.parts))
path_parts = list(reversed(path.parts)) path_parts = list(reversed(path.parts))
@ -33,9 +33,9 @@ class NormalRule(Rule):
if left_parts: if left_parts:
return None return None
return Path(*path_parts) return PurePath(*path_parts)
def transform(self, path: Path) -> Optional[Path]: def transform(self, path: PurePath) -> Optional[PurePath]:
if rest := self._match_prefix(path): if rest := self._match_prefix(path):
return self._right / rest return self._right / rest
@ -43,11 +43,11 @@ class NormalRule(Rule):
class ExactRule(Rule): class ExactRule(Rule):
def __init__(self, left: Path, right: Path): def __init__(self, left: PurePath, right: PurePath):
self._left = left self._left = left
self._right = right self._right = right
def transform(self, path: Path) -> Optional[Path]: def transform(self, path: PurePath) -> Optional[PurePath]:
if path == self._left: if path == self._left:
return self._right return self._right
@ -59,7 +59,7 @@ class ReRule(Rule):
self._left = left self._left = left
self._right = right self._right = right
def transform(self, path: Path) -> Optional[Path]: def transform(self, path: PurePath) -> Optional[PurePath]:
if match := re.fullmatch(self._left, str(path)): if match := re.fullmatch(self._left, str(path)):
kwargs: Dict[str, Union[int, float]] = {} kwargs: Dict[str, Union[int, float]] = {}
@ -75,7 +75,7 @@ class ReRule(Rule):
except ValueError: except ValueError:
pass pass
return Path(self._right.format(*groups, **kwargs)) return PurePath(self._right.format(*groups, **kwargs))
return None return None
@ -208,9 +208,9 @@ def parse_rule(line: Line) -> Rule:
right = parse_string(line) right = parse_string(line)
if arrowname == "": if arrowname == "":
return NormalRule(Path(left), Path(right)) return NormalRule(PurePath(left), PurePath(right))
elif arrowname == "exact": elif arrowname == "exact":
return ExactRule(Path(left), Path(right)) return ExactRule(PurePath(left), PurePath(right))
elif arrowname == "re": elif arrowname == "re":
return ReRule(left, right) return ReRule(left, right)
else: else:
@ -230,7 +230,7 @@ class Transformer:
if line: if line:
self._rules.append(parse_rule(Line(line, i))) self._rules.append(parse_rule(Line(line, i)))
def transform(self, path: Path) -> Optional[Path]: def transform(self, path: PurePath) -> Optional[PurePath]:
for rule in self._rules: for rule in self._rules:
if result := rule.transform(path): if result := rule.transform(path):
return result return result