common: load site list from yaml, remove cache crap
This commit is contained in:
parent
ae0024f343
commit
3194388616
|
@ -1,31 +1,26 @@
|
||||||
"""Cache management"""
|
"""Cache management"""
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from pickle import dumps, loads
|
|
||||||
from shutil import rmtree
|
from shutil import rmtree
|
||||||
from typing import Awaitable, Callable, Generic, TypeVar, Union, cast
|
from typing import Awaitable, Callable, Union
|
||||||
|
|
||||||
from click import echo
|
from click import echo
|
||||||
from xdg import xdg_cache_home
|
from xdg import xdg_cache_home
|
||||||
|
|
||||||
ResourceType = TypeVar("ResourceType")
|
CacheFallback = Union[bytes, Callable[[str], Awaitable[bytes]]]
|
||||||
|
|
||||||
CacheFallback = Union[ResourceType, Callable[[str], Awaitable[ResourceType]]]
|
|
||||||
|
|
||||||
|
|
||||||
class Cache(Generic[ResourceType], ABC):
|
class Cache(ABC):
|
||||||
"""Base class for caches"""
|
"""Base class for caches"""
|
||||||
|
|
||||||
cache_base = xdg_cache_home() / "frontools"
|
cache_base = xdg_cache_home() / "frontools"
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def get(
|
async def get(self, key: str, fallback: CacheFallback) -> bytes:
|
||||||
self, key: str, fallback: CacheFallback[ResourceType]
|
|
||||||
) -> ResourceType:
|
|
||||||
"""Get an item in the cache, call fallback if it's not present"""
|
"""Get an item in the cache, call fallback if it's not present"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def set(self, key: str, resource: ResourceType) -> None:
|
def set(self, key: str, data: bytes) -> None:
|
||||||
"""Set a resource in the cache"""
|
"""Set a resource in the cache"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -45,9 +40,7 @@ class Cache(Generic[ResourceType], ABC):
|
||||||
rmtree(cache_path)
|
rmtree(cache_path)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def _get_fallback_value(
|
async def _get_fallback_value(key: str, fallback: CacheFallback) -> bytes:
|
||||||
key: str, fallback: CacheFallback[ResourceType]
|
|
||||||
) -> ResourceType:
|
|
||||||
if callable(fallback):
|
if callable(fallback):
|
||||||
result = await fallback(key)
|
result = await fallback(key)
|
||||||
else:
|
else:
|
||||||
|
@ -56,19 +49,17 @@ class Cache(Generic[ResourceType], ABC):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
class NullCache(Cache[ResourceType]):
|
class NullCache(Cache):
|
||||||
"""Disabled cache"""
|
"""Disabled cache"""
|
||||||
|
|
||||||
async def get(
|
async def get(self, key: str, fallback: CacheFallback) -> bytes:
|
||||||
self, key: str, fallback: CacheFallback[ResourceType]
|
|
||||||
) -> ResourceType:
|
|
||||||
return await self._get_fallback_value(key, fallback)
|
return await self._get_fallback_value(key, fallback)
|
||||||
|
|
||||||
def set(self, key: str, resource: ResourceType) -> None:
|
def set(self, key: str, resource: bytes) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class FileCache(Cache[ResourceType]):
|
class FileCache(Cache):
|
||||||
"""Cache on the local filesystem"""
|
"""Cache on the local filesystem"""
|
||||||
|
|
||||||
def __init__(self, name: str) -> None:
|
def __init__(self, name: str) -> None:
|
||||||
|
@ -77,8 +68,8 @@ class FileCache(Cache[ResourceType]):
|
||||||
async def get(
|
async def get(
|
||||||
self,
|
self,
|
||||||
key: str,
|
key: str,
|
||||||
fallback: CacheFallback[ResourceType],
|
fallback: CacheFallback,
|
||||||
) -> ResourceType:
|
) -> bytes:
|
||||||
"""Get an item in the cache, call fallback if it's not present"""
|
"""Get an item in the cache, call fallback if it's not present"""
|
||||||
cache_file_path = self._get_cache_file_path(key)
|
cache_file_path = self._get_cache_file_path(key)
|
||||||
if not cache_file_path.is_file():
|
if not cache_file_path.is_file():
|
||||||
|
@ -86,17 +77,15 @@ class FileCache(Cache[ResourceType]):
|
||||||
self.set(key, resource)
|
self.set(key, resource)
|
||||||
else:
|
else:
|
||||||
with open(cache_file_path, "rb") as cache_file:
|
with open(cache_file_path, "rb") as cache_file:
|
||||||
resource_data = cache_file.read()
|
resource = cache_file.read()
|
||||||
resource = self._deserialize(resource_data)
|
|
||||||
|
|
||||||
return resource
|
return resource
|
||||||
|
|
||||||
def set(self, key: str, resource: ResourceType) -> None:
|
def set(self, key: str, resource: bytes) -> None:
|
||||||
"""Set a resource in the cache"""
|
"""Set a resource in the cache"""
|
||||||
cache_file_path = self._get_cache_file_path(key)
|
cache_file_path = self._get_cache_file_path(key)
|
||||||
with open(cache_file_path, "wb") as cache_file:
|
with open(cache_file_path, "wb") as cache_file:
|
||||||
resource_data = self._serialize(resource)
|
cache_file.write(resource)
|
||||||
cache_file.write(resource_data)
|
|
||||||
|
|
||||||
def _get_cache_file_path(self, key: str) -> Path:
|
def _get_cache_file_path(self, key: str) -> Path:
|
||||||
key_slug = _get_key_slug(key)
|
key_slug = _get_key_slug(key)
|
||||||
|
@ -110,34 +99,6 @@ class FileCache(Cache[ResourceType]):
|
||||||
|
|
||||||
return file_path
|
return file_path
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def _serialize(self, resource: ResourceType) -> bytes:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def _deserialize(self, data: bytes) -> ResourceType:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class DataCache(FileCache[bytes]):
|
|
||||||
"""Cache of byte array"""
|
|
||||||
|
|
||||||
def _serialize(self, resource: bytes) -> bytes:
|
|
||||||
return resource
|
|
||||||
|
|
||||||
def _deserialize(self, data: bytes) -> bytes:
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
class ObjectCache(FileCache[ResourceType]):
|
|
||||||
"""Cache pickling objects"""
|
|
||||||
|
|
||||||
def _serialize(self, resource: ResourceType) -> bytes:
|
|
||||||
return dumps(resource)
|
|
||||||
|
|
||||||
def _deserialize(self, data: bytes) -> ResourceType:
|
|
||||||
return cast(ResourceType, loads(data))
|
|
||||||
|
|
||||||
|
|
||||||
def _get_key_slug(url: str) -> str:
|
def _get_key_slug(url: str) -> str:
|
||||||
"""Return an unique slug usable as a path name for a given url."""
|
"""Return an unique slug usable as a path name for a given url."""
|
||||||
|
|
|
@ -4,11 +4,13 @@ from importlib.util import module_from_spec, spec_from_file_location
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from re import Pattern
|
from re import Pattern
|
||||||
from re import compile as re_compile
|
from re import compile as re_compile
|
||||||
from typing import Iterable, Optional
|
from typing import Any, Iterable, Optional, Type
|
||||||
|
|
||||||
from xdg import xdg_config_dirs
|
from xdg import xdg_config_dirs
|
||||||
|
from yaml import Loader
|
||||||
|
from yaml import load as load_yaml
|
||||||
|
|
||||||
from frontools.cache import Cache, DataCache, NullCache, ObjectCache
|
from frontools.cache import Cache, FileCache, NullCache
|
||||||
from frontools.sources import CachedSource, OverrideSource, Source
|
from frontools.sources import CachedSource, OverrideSource, Source
|
||||||
from frontools.utils import ErrorSummary
|
from frontools.utils import ErrorSummary
|
||||||
|
|
||||||
|
@ -49,11 +51,11 @@ class Config:
|
||||||
self._default_source_name = default_source_name
|
self._default_source_name = default_source_name
|
||||||
|
|
||||||
self._error_summary = ErrorSummary()
|
self._error_summary = ErrorSummary()
|
||||||
remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME)
|
if use_cache:
|
||||||
self._add_source(
|
remote_cache: Cache = FileCache(REMOTE_SOURCE_NAME)
|
||||||
REMOTE_SOURCE_NAME,
|
else:
|
||||||
CachedSource(self._error_summary, self._block_urls, remote_cache),
|
remote_cache = NullCache()
|
||||||
)
|
self._add_source(REMOTE_SOURCE_NAME, CachedSource, remote_cache)
|
||||||
self._include_urls = [re_compile(it) for it in include_urls]
|
self._include_urls = [re_compile(it) for it in include_urls]
|
||||||
self._exclude_urls = [re_compile(it) for it in exclude_urls]
|
self._exclude_urls = [re_compile(it) for it in exclude_urls]
|
||||||
|
|
||||||
|
@ -106,11 +108,6 @@ class Config:
|
||||||
"""Return sites configured for this context"""
|
"""Return sites configured for this context"""
|
||||||
return self._sites.items()
|
return self._sites.items()
|
||||||
|
|
||||||
@property
|
|
||||||
def config_cache(self) -> Cache[object]:
|
|
||||||
"""Get the cache for configuration"""
|
|
||||||
return self.get_object_cache("config")
|
|
||||||
|
|
||||||
def add_site_url(self, name: str, url: str) -> None:
|
def add_site_url(self, name: str, url: str) -> None:
|
||||||
"""Add an url for a site"""
|
"""Add an url for a site"""
|
||||||
if len(self._include_urls):
|
if len(self._include_urls):
|
||||||
|
@ -126,25 +123,19 @@ class Config:
|
||||||
|
|
||||||
self._sites[name].urls.append(url)
|
self._sites[name].urls.append(url)
|
||||||
|
|
||||||
|
def load_sites_from_yaml(self, yaml_path: str) -> None:
|
||||||
|
"""Load a yaml file containing dictionnary of urls to add as sites."""
|
||||||
|
with open(yaml_path, "r", encoding="utf-8") as yaml_file:
|
||||||
|
yaml_document = load_yaml(yaml_file, Loader)
|
||||||
|
for site_name, urls in yaml_document.items():
|
||||||
|
for url in urls:
|
||||||
|
self.add_site_url(site_name, url)
|
||||||
|
|
||||||
def block_url_patterns(self, *patterns: str) -> None:
|
def block_url_patterns(self, *patterns: str) -> None:
|
||||||
"""Will return 500 error for urls matching this pattern."""
|
"""Will return 500 error for urls matching this pattern."""
|
||||||
for pattern in patterns:
|
for pattern in patterns:
|
||||||
self._block_urls.append(re_compile(pattern))
|
self._block_urls.append(re_compile(pattern))
|
||||||
|
|
||||||
def get_data_cache(self, name: str) -> Cache[bytes]:
|
|
||||||
"""Get a data cache with the given identifier"""
|
|
||||||
if self._use_cache:
|
|
||||||
return DataCache(name)
|
|
||||||
|
|
||||||
return NullCache()
|
|
||||||
|
|
||||||
def get_object_cache(self, name: str) -> Cache[object]:
|
|
||||||
"""Get an object cache with the given identifier"""
|
|
||||||
if self._use_cache:
|
|
||||||
return ObjectCache[object](name)
|
|
||||||
|
|
||||||
return NullCache()
|
|
||||||
|
|
||||||
def add_override_source(
|
def add_override_source(
|
||||||
self,
|
self,
|
||||||
name: str,
|
name: str,
|
||||||
|
@ -157,9 +148,7 @@ class Config:
|
||||||
next_source = self.default_source
|
next_source = self.default_source
|
||||||
else:
|
else:
|
||||||
next_source = self.get_source(next_source_name)
|
next_source = self.get_source(next_source_name)
|
||||||
self._sources[name] = OverrideSource(
|
self._add_source(name, OverrideSource, mappings, next_source)
|
||||||
self._error_summary, self._block_urls, mappings, next_source
|
|
||||||
)
|
|
||||||
|
|
||||||
def get_source(self, name: str) -> Source:
|
def get_source(self, name: str) -> Source:
|
||||||
"""Get an alternate source in the configured ones"""
|
"""Get an alternate source in the configured ones"""
|
||||||
|
@ -170,10 +159,14 @@ class Config:
|
||||||
def echo_error_summary(self) -> None:
|
def echo_error_summary(self) -> None:
|
||||||
self._error_summary.echo()
|
self._error_summary.echo()
|
||||||
|
|
||||||
def _add_source(self, name: str, source: Source) -> None:
|
def _add_source(
|
||||||
|
self, name: str, source_class: Type[Source], *args: Any, **kwargs: Any
|
||||||
|
) -> None:
|
||||||
if name in self._sources:
|
if name in self._sources:
|
||||||
raise Exception(f"Source {name} already configured")
|
raise Exception(f"Source {name} already configured")
|
||||||
self._sources[name] = source
|
self._sources[name] = source_class(
|
||||||
|
self._error_summary, self._block_urls, *args, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _find_config() -> Optional[Path]:
|
def _find_config() -> Optional[Path]:
|
||||||
|
|
|
@ -104,7 +104,7 @@ class CachedSource(Source):
|
||||||
self,
|
self,
|
||||||
error_summary: ErrorSummary,
|
error_summary: ErrorSummary,
|
||||||
block_urls: list[Pattern[str]],
|
block_urls: list[Pattern[str]],
|
||||||
cache: Cache[bytes],
|
cache: Cache,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(error_summary, block_urls)
|
super().__init__(error_summary, block_urls)
|
||||||
self._cache = cache
|
self._cache = cache
|
||||||
|
|
|
@ -44,7 +44,7 @@ def isort(session: Session) -> None:
|
||||||
@nox.session(reuse_venv=True)
|
@nox.session(reuse_venv=True)
|
||||||
def mypy(session: Session) -> None:
|
def mypy(session: Session) -> None:
|
||||||
"""Run Mypy"""
|
"""Run Mypy"""
|
||||||
session.install("mypy", "types-click", "-e", ".")
|
session.install("mypy", "types-pyyaml", "types-click", "-e", ".")
|
||||||
session.run("mypy", "--install-types", *LINT_PATHS)
|
session.run("mypy", "--install-types", *LINT_PATHS)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -47,6 +47,7 @@ mypy = "^0.942"
|
||||||
pylint = "^2.13.2"
|
pylint = "^2.13.2"
|
||||||
types-click = "^7.1.8"
|
types-click = "^7.1.8"
|
||||||
types-Pillow = "^9.0.11"
|
types-Pillow = "^9.0.11"
|
||||||
|
types-PyYAML = "^6.0.5"
|
||||||
|
|
||||||
[tool.poetry-dynamic-versioning]
|
[tool.poetry-dynamic-versioning]
|
||||||
enable = true
|
enable = true
|
||||||
|
|
Loading…
Reference in New Issue