mirror of
https://github.com/Garmelon/PFERD.git
synced 2023-12-21 10:23:01 +01:00
Use PurePath instead of Path
Path should only be used when we need to access the file system. For all other purposes (mainly crawling), we use PurePath instead since the paths don't correspond to paths in the local file system.
This commit is contained in:
parent
0096d83387
commit
f776186480
@ -1,7 +1,7 @@
|
|||||||
import configparser
|
import configparser
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from pathlib import Path
|
from pathlib import PurePath
|
||||||
# TODO In Python 3.9 and above, AsyncContextManager is deprecated
|
# TODO In Python 3.9 and above, AsyncContextManager is deprecated
|
||||||
from typing import AsyncContextManager, AsyncIterator, Optional
|
from typing import AsyncContextManager, AsyncIterator, Optional
|
||||||
|
|
||||||
@ -38,7 +38,8 @@ class Crawler(ABC):
|
|||||||
e.pretty_print()
|
e.pretty_print()
|
||||||
raise CrawlerLoadException()
|
raise CrawlerLoadException()
|
||||||
|
|
||||||
# output_dir = Path(section.get("output_dir", name))
|
# working_dir = Path(section.get("working_dir", ""))
|
||||||
|
# output_dir = working_dir / section.get("output_dir", name)
|
||||||
|
|
||||||
def print(self, text: str) -> None:
|
def print(self, text: str) -> None:
|
||||||
"""
|
"""
|
||||||
@ -75,14 +76,14 @@ class Crawler(ABC):
|
|||||||
with self._conductor.progress_bar(desc, total=total) as bar:
|
with self._conductor.progress_bar(desc, total=total) as bar:
|
||||||
yield bar
|
yield bar
|
||||||
|
|
||||||
def crawl_bar(self, path: Path) -> AsyncContextManager[ProgressBar]:
|
def crawl_bar(self, path: PurePath) -> AsyncContextManager[ProgressBar]:
|
||||||
pathstr = escape(str(path))
|
pathstr = escape(str(path))
|
||||||
desc = f"[bold magenta]Crawling[/bold magenta] {pathstr}"
|
desc = f"[bold magenta]Crawling[/bold magenta] {pathstr}"
|
||||||
return self.progress_bar(desc)
|
return self.progress_bar(desc)
|
||||||
|
|
||||||
def download_bar(
|
def download_bar(
|
||||||
self,
|
self,
|
||||||
path: Path,
|
path: PurePath,
|
||||||
size: int,
|
size: int,
|
||||||
) -> AsyncContextManager[ProgressBar]:
|
) -> AsyncContextManager[ProgressBar]:
|
||||||
pathstr = escape(str(path))
|
pathstr = escape(str(path))
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import random
|
import random
|
||||||
from pathlib import Path
|
from pathlib import PurePath
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from rich.markup import escape
|
from rich.markup import escape
|
||||||
@ -37,9 +37,9 @@ DUMMY_TREE = {
|
|||||||
|
|
||||||
class DummyCrawler(Crawler):
|
class DummyCrawler(Crawler):
|
||||||
async def crawl(self) -> None:
|
async def crawl(self) -> None:
|
||||||
await self._crawl_entry(Path(), DUMMY_TREE)
|
await self._crawl_entry(PurePath(), DUMMY_TREE)
|
||||||
|
|
||||||
async def _crawl_entry(self, path: Path, value: Any) -> None:
|
async def _crawl_entry(self, path: PurePath, value: Any) -> None:
|
||||||
if value is True:
|
if value is True:
|
||||||
async with self.exclusive_output():
|
async with self.exclusive_output():
|
||||||
await ainput(f"File {path}, please press enter: ")
|
await ainput(f"File {path}, please press enter: ")
|
||||||
|
@ -1,22 +1,22 @@
|
|||||||
import re
|
import re
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import PurePath
|
||||||
from typing import Dict, Optional, Union
|
from typing import Dict, Optional, Union
|
||||||
|
|
||||||
|
|
||||||
class Rule(ABC):
|
class Rule(ABC):
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def transform(self, path: Path) -> Optional[Path]:
|
def transform(self, path: PurePath) -> Optional[PurePath]:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class NormalRule(Rule):
|
class NormalRule(Rule):
|
||||||
def __init__(self, left: Path, right: Path):
|
def __init__(self, left: PurePath, right: PurePath):
|
||||||
self._left = left
|
self._left = left
|
||||||
self._right = right
|
self._right = right
|
||||||
|
|
||||||
def _match_prefix(self, path: Path) -> Optional[Path]:
|
def _match_prefix(self, path: PurePath) -> Optional[PurePath]:
|
||||||
left_parts = list(reversed(self._left.parts))
|
left_parts = list(reversed(self._left.parts))
|
||||||
path_parts = list(reversed(path.parts))
|
path_parts = list(reversed(path.parts))
|
||||||
|
|
||||||
@ -33,9 +33,9 @@ class NormalRule(Rule):
|
|||||||
if left_parts:
|
if left_parts:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return Path(*path_parts)
|
return PurePath(*path_parts)
|
||||||
|
|
||||||
def transform(self, path: Path) -> Optional[Path]:
|
def transform(self, path: PurePath) -> Optional[PurePath]:
|
||||||
if rest := self._match_prefix(path):
|
if rest := self._match_prefix(path):
|
||||||
return self._right / rest
|
return self._right / rest
|
||||||
|
|
||||||
@ -43,11 +43,11 @@ class NormalRule(Rule):
|
|||||||
|
|
||||||
|
|
||||||
class ExactRule(Rule):
|
class ExactRule(Rule):
|
||||||
def __init__(self, left: Path, right: Path):
|
def __init__(self, left: PurePath, right: PurePath):
|
||||||
self._left = left
|
self._left = left
|
||||||
self._right = right
|
self._right = right
|
||||||
|
|
||||||
def transform(self, path: Path) -> Optional[Path]:
|
def transform(self, path: PurePath) -> Optional[PurePath]:
|
||||||
if path == self._left:
|
if path == self._left:
|
||||||
return self._right
|
return self._right
|
||||||
|
|
||||||
@ -59,7 +59,7 @@ class ReRule(Rule):
|
|||||||
self._left = left
|
self._left = left
|
||||||
self._right = right
|
self._right = right
|
||||||
|
|
||||||
def transform(self, path: Path) -> Optional[Path]:
|
def transform(self, path: PurePath) -> Optional[PurePath]:
|
||||||
if match := re.fullmatch(self._left, str(path)):
|
if match := re.fullmatch(self._left, str(path)):
|
||||||
kwargs: Dict[str, Union[int, float]] = {}
|
kwargs: Dict[str, Union[int, float]] = {}
|
||||||
|
|
||||||
@ -75,7 +75,7 @@ class ReRule(Rule):
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
return Path(self._right.format(*groups, **kwargs))
|
return PurePath(self._right.format(*groups, **kwargs))
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@ -208,9 +208,9 @@ def parse_rule(line: Line) -> Rule:
|
|||||||
right = parse_string(line)
|
right = parse_string(line)
|
||||||
|
|
||||||
if arrowname == "":
|
if arrowname == "":
|
||||||
return NormalRule(Path(left), Path(right))
|
return NormalRule(PurePath(left), PurePath(right))
|
||||||
elif arrowname == "exact":
|
elif arrowname == "exact":
|
||||||
return ExactRule(Path(left), Path(right))
|
return ExactRule(PurePath(left), PurePath(right))
|
||||||
elif arrowname == "re":
|
elif arrowname == "re":
|
||||||
return ReRule(left, right)
|
return ReRule(left, right)
|
||||||
else:
|
else:
|
||||||
@ -230,7 +230,7 @@ class Transformer:
|
|||||||
if line:
|
if line:
|
||||||
self._rules.append(parse_rule(Line(line, i)))
|
self._rules.append(parse_rule(Line(line, i)))
|
||||||
|
|
||||||
def transform(self, path: Path) -> Optional[Path]:
|
def transform(self, path: PurePath) -> Optional[PurePath]:
|
||||||
for rule in self._rules:
|
for rule in self._rules:
|
||||||
if result := rule.transform(path):
|
if result := rule.transform(path):
|
||||||
return result
|
return result
|
||||||
|
Loading…
Reference in New Issue
Block a user