common: load site list from yaml, remove cache crap
This commit is contained in:
parent
ae0024f343
commit
3194388616
|
|
@ -1,31 +1,26 @@
|
|||
"""Cache management"""
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from pickle import dumps, loads
|
||||
from shutil import rmtree
|
||||
from typing import Awaitable, Callable, Generic, TypeVar, Union, cast
|
||||
from typing import Awaitable, Callable, Union
|
||||
|
||||
from click import echo
|
||||
from xdg import xdg_cache_home
|
||||
|
||||
ResourceType = TypeVar("ResourceType")
|
||||
|
||||
CacheFallback = Union[ResourceType, Callable[[str], Awaitable[ResourceType]]]
|
||||
CacheFallback = Union[bytes, Callable[[str], Awaitable[bytes]]]
|
||||
|
||||
|
||||
class Cache(Generic[ResourceType], ABC):
|
||||
class Cache(ABC):
|
||||
"""Base class for caches"""
|
||||
|
||||
cache_base = xdg_cache_home() / "frontools"
|
||||
|
||||
@abstractmethod
|
||||
async def get(
|
||||
self, key: str, fallback: CacheFallback[ResourceType]
|
||||
) -> ResourceType:
|
||||
async def get(self, key: str, fallback: CacheFallback) -> bytes:
|
||||
"""Get an item in the cache, call fallback if it's not present"""
|
||||
|
||||
@abstractmethod
|
||||
def set(self, key: str, resource: ResourceType) -> None:
|
||||
def set(self, key: str, data: bytes) -> None:
|
||||
"""Set a resource in the cache"""
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -45,9 +40,7 @@ class Cache(Generic[ResourceType], ABC):
|
|||
rmtree(cache_path)
|
||||
|
||||
@staticmethod
|
||||
async def _get_fallback_value(
|
||||
key: str, fallback: CacheFallback[ResourceType]
|
||||
) -> ResourceType:
|
||||
async def _get_fallback_value(key: str, fallback: CacheFallback) -> bytes:
|
||||
if callable(fallback):
|
||||
result = await fallback(key)
|
||||
else:
|
||||
|
|
@ -56,19 +49,17 @@ class Cache(Generic[ResourceType], ABC):
|
|||
return result
|
||||
|
||||
|
||||
class NullCache(Cache[ResourceType]):
|
||||
class NullCache(Cache):
|
||||
"""Disabled cache"""
|
||||
|
||||
async def get(
|
||||
self, key: str, fallback: CacheFallback[ResourceType]
|
||||
) -> ResourceType:
|
||||
async def get(self, key: str, fallback: CacheFallback) -> bytes:
|
||||
return await self._get_fallback_value(key, fallback)
|
||||
|
||||
def set(self, key: str, resource: ResourceType) -> None:
|
||||
def set(self, key: str, resource: bytes) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class FileCache(Cache[ResourceType]):
|
||||
class FileCache(Cache):
|
||||
"""Cache on the local filesystem"""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
|
|
@ -77,8 +68,8 @@ class FileCache(Cache[ResourceType]):
|
|||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
fallback: CacheFallback[ResourceType],
|
||||
) -> ResourceType:
|
||||
fallback: CacheFallback,
|
||||
) -> bytes:
|
||||
"""Get an item in the cache, call fallback if it's not present"""
|
||||
cache_file_path = self._get_cache_file_path(key)
|
||||
if not cache_file_path.is_file():
|
||||
|
|
@ -86,17 +77,15 @@ class FileCache(Cache[ResourceType]):
|
|||
self.set(key, resource)
|
||||
else:
|
||||
with open(cache_file_path, "rb") as cache_file:
|
||||
resource_data = cache_file.read()
|
||||
resource = self._deserialize(resource_data)
|
||||
resource = cache_file.read()
|
||||
|
||||
return resource
|
||||
|
||||
def set(self, key: str, resource: ResourceType) -> None:
|
||||
def set(self, key: str, resource: bytes) -> None:
|
||||
"""Set a resource in the cache"""
|
||||
cache_file_path = self._get_cache_file_path(key)
|
||||
with open(cache_file_path, "wb") as cache_file:
|
||||
resource_data = self._serialize(resource)
|
||||
cache_file.write(resource_data)
|
||||
cache_file.write(resource)
|
||||
|
||||
def _get_cache_file_path(self, key: str) -> Path:
|
||||
key_slug = _get_key_slug(key)
|
||||
|
|
@ -110,34 +99,6 @@ class FileCache(Cache[ResourceType]):
|
|||
|
||||
return file_path
|
||||
|
||||
@abstractmethod
|
||||
def _serialize(self, resource: ResourceType) -> bytes:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _deserialize(self, data: bytes) -> ResourceType:
|
||||
pass
|
||||
|
||||
|
||||
class DataCache(FileCache[bytes]):
|
||||
"""Cache of byte array"""
|
||||
|
||||
def _serialize(self, resource: bytes) -> bytes:
|
||||
return resource
|
||||
|
||||
def _deserialize(self, data: bytes) -> bytes:
|
||||
return data
|
||||
|
||||
|
||||
class ObjectCache(FileCache[ResourceType]):
|
||||
"""Cache pickling objects"""
|
||||
|
||||
def _serialize(self, resource: ResourceType) -> bytes:
|
||||
return dumps(resource)
|
||||
|
||||
def _deserialize(self, data: bytes) -> ResourceType:
|
||||
return cast(ResourceType, loads(data))
|
||||
|
||||
|
||||
def _get_key_slug(url: str) -> str:
|
||||
"""Return an unique slug usable as a path name for a given url."""
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@ from importlib.util import module_from_spec, spec_from_file_location
|
|||
from pathlib import Path
|
||||
from re import Pattern
|
||||
from re import compile as re_compile
|
||||
from typing import Iterable, Optional
|
||||
from typing import Any, Iterable, Optional, Type
|
||||
|
||||
from xdg import xdg_config_dirs
|
||||
from yaml import Loader
|
||||
from yaml import load as load_yaml
|
||||
|
||||
from frontools.cache import Cache, DataCache, NullCache, ObjectCache
|
||||
from frontools.cache import Cache, FileCache, NullCache
|
||||
from frontools.sources import CachedSource, OverrideSource, Source
|
||||
from frontools.utils import ErrorSummary
|
||||
|
||||
|
|
@ -49,11 +51,11 @@ class Config:
|
|||
self._default_source_name = default_source_name
|
||||
|
||||
self._error_summary = ErrorSummary()
|
||||
remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME)
|
||||
self._add_source(
|
||||
REMOTE_SOURCE_NAME,
|
||||
CachedSource(self._error_summary, self._block_urls, remote_cache),
|
||||
)
|
||||
if use_cache:
|
||||
remote_cache: Cache = FileCache(REMOTE_SOURCE_NAME)
|
||||
else:
|
||||
remote_cache = NullCache()
|
||||
self._add_source(REMOTE_SOURCE_NAME, CachedSource, remote_cache)
|
||||
self._include_urls = [re_compile(it) for it in include_urls]
|
||||
self._exclude_urls = [re_compile(it) for it in exclude_urls]
|
||||
|
||||
|
|
@ -106,11 +108,6 @@ class Config:
|
|||
"""Return sites configured for this context"""
|
||||
return self._sites.items()
|
||||
|
||||
@property
|
||||
def config_cache(self) -> Cache[object]:
|
||||
"""Get the cache for configuration"""
|
||||
return self.get_object_cache("config")
|
||||
|
||||
def add_site_url(self, name: str, url: str) -> None:
|
||||
"""Add an url for a site"""
|
||||
if len(self._include_urls):
|
||||
|
|
@ -126,25 +123,19 @@ class Config:
|
|||
|
||||
self._sites[name].urls.append(url)
|
||||
|
||||
def load_sites_from_yaml(self, yaml_path: str) -> None:
|
||||
"""Load a yaml file containing dictionnary of urls to add as sites."""
|
||||
with open(yaml_path, "r", encoding="utf-8") as yaml_file:
|
||||
yaml_document = load_yaml(yaml_file, Loader)
|
||||
for site_name, urls in yaml_document.items():
|
||||
for url in urls:
|
||||
self.add_site_url(site_name, url)
|
||||
|
||||
def block_url_patterns(self, *patterns: str) -> None:
|
||||
"""Will return 500 error for urls matching this pattern."""
|
||||
for pattern in patterns:
|
||||
self._block_urls.append(re_compile(pattern))
|
||||
|
||||
def get_data_cache(self, name: str) -> Cache[bytes]:
|
||||
"""Get a data cache with the given identifier"""
|
||||
if self._use_cache:
|
||||
return DataCache(name)
|
||||
|
||||
return NullCache()
|
||||
|
||||
def get_object_cache(self, name: str) -> Cache[object]:
|
||||
"""Get an object cache with the given identifier"""
|
||||
if self._use_cache:
|
||||
return ObjectCache[object](name)
|
||||
|
||||
return NullCache()
|
||||
|
||||
def add_override_source(
|
||||
self,
|
||||
name: str,
|
||||
|
|
@ -157,9 +148,7 @@ class Config:
|
|||
next_source = self.default_source
|
||||
else:
|
||||
next_source = self.get_source(next_source_name)
|
||||
self._sources[name] = OverrideSource(
|
||||
self._error_summary, self._block_urls, mappings, next_source
|
||||
)
|
||||
self._add_source(name, OverrideSource, mappings, next_source)
|
||||
|
||||
def get_source(self, name: str) -> Source:
|
||||
"""Get an alternate source in the configured ones"""
|
||||
|
|
@ -170,10 +159,14 @@ class Config:
|
|||
def echo_error_summary(self) -> None:
|
||||
self._error_summary.echo()
|
||||
|
||||
def _add_source(self, name: str, source: Source) -> None:
|
||||
def _add_source(
|
||||
self, name: str, source_class: Type[Source], *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
if name in self._sources:
|
||||
raise Exception(f"Source {name} already configured")
|
||||
self._sources[name] = source
|
||||
self._sources[name] = source_class(
|
||||
self._error_summary, self._block_urls, *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def _find_config() -> Optional[Path]:
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ class CachedSource(Source):
|
|||
self,
|
||||
error_summary: ErrorSummary,
|
||||
block_urls: list[Pattern[str]],
|
||||
cache: Cache[bytes],
|
||||
cache: Cache,
|
||||
) -> None:
|
||||
super().__init__(error_summary, block_urls)
|
||||
self._cache = cache
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ def isort(session: Session) -> None:
|
|||
@nox.session(reuse_venv=True)
|
||||
def mypy(session: Session) -> None:
|
||||
"""Run Mypy"""
|
||||
session.install("mypy", "types-click", "-e", ".")
|
||||
session.install("mypy", "types-pyyaml", "types-click", "-e", ".")
|
||||
session.run("mypy", "--install-types", *LINT_PATHS)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ mypy = "^0.942"
|
|||
pylint = "^2.13.2"
|
||||
types-click = "^7.1.8"
|
||||
types-Pillow = "^9.0.11"
|
||||
types-PyYAML = "^6.0.5"
|
||||
|
||||
[tool.poetry-dynamic-versioning]
|
||||
enable = true
|
||||
|
|
|
|||
Loading…
Reference in New Issue