From 31943886160e55844915cea616bd50298a420cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Corentin=20S=C3=A9chet?= Date: Mon, 11 Apr 2022 17:31:16 +0200 Subject: [PATCH] common: load site list from yaml, remove cache crap --- frontools/cache.py | 69 ++++++++++---------------------------------- frontools/config.py | 55 +++++++++++++++-------------------- frontools/sources.py | 2 +- noxfile.py | 2 +- pyproject.toml | 1 + 5 files changed, 42 insertions(+), 87 deletions(-) diff --git a/frontools/cache.py b/frontools/cache.py index 6315ec0..6d5db4c 100644 --- a/frontools/cache.py +++ b/frontools/cache.py @@ -1,31 +1,26 @@ """Cache management""" from abc import ABC, abstractmethod from pathlib import Path -from pickle import dumps, loads from shutil import rmtree -from typing import Awaitable, Callable, Generic, TypeVar, Union, cast +from typing import Awaitable, Callable, Union from click import echo from xdg import xdg_cache_home -ResourceType = TypeVar("ResourceType") - -CacheFallback = Union[ResourceType, Callable[[str], Awaitable[ResourceType]]] +CacheFallback = Union[bytes, Callable[[str], Awaitable[bytes]]] -class Cache(Generic[ResourceType], ABC): +class Cache(ABC): """Base class for caches""" cache_base = xdg_cache_home() / "frontools" @abstractmethod - async def get( - self, key: str, fallback: CacheFallback[ResourceType] - ) -> ResourceType: + async def get(self, key: str, fallback: CacheFallback) -> bytes: """Get an item in the cache, call fallback if it's not present""" @abstractmethod - def set(self, key: str, resource: ResourceType) -> None: + def set(self, key: str, data: bytes) -> None: """Set a resource in the cache""" @staticmethod @@ -45,9 +40,7 @@ class Cache(Generic[ResourceType], ABC): rmtree(cache_path) @staticmethod - async def _get_fallback_value( - key: str, fallback: CacheFallback[ResourceType] - ) -> ResourceType: + async def _get_fallback_value(key: str, fallback: CacheFallback) -> bytes: if callable(fallback): result = await fallback(key) else: @@ -56,19 +49,17 @@ class Cache(Generic[ResourceType], ABC): return result -class NullCache(Cache[ResourceType]): +class NullCache(Cache): """Disabled cache""" - async def get( - self, key: str, fallback: CacheFallback[ResourceType] - ) -> ResourceType: + async def get(self, key: str, fallback: CacheFallback) -> bytes: return await self._get_fallback_value(key, fallback) - def set(self, key: str, resource: ResourceType) -> None: + def set(self, key: str, resource: bytes) -> None: pass -class FileCache(Cache[ResourceType]): +class FileCache(Cache): """Cache on the local filesystem""" def __init__(self, name: str) -> None: @@ -77,8 +68,8 @@ class FileCache(Cache[ResourceType]): async def get( self, key: str, - fallback: CacheFallback[ResourceType], - ) -> ResourceType: + fallback: CacheFallback, + ) -> bytes: """Get an item in the cache, call fallback if it's not present""" cache_file_path = self._get_cache_file_path(key) if not cache_file_path.is_file(): @@ -86,17 +77,15 @@ class FileCache(Cache[ResourceType]): self.set(key, resource) else: with open(cache_file_path, "rb") as cache_file: - resource_data = cache_file.read() - resource = self._deserialize(resource_data) + resource = cache_file.read() return resource - def set(self, key: str, resource: ResourceType) -> None: + def set(self, key: str, resource: bytes) -> None: """Set a resource in the cache""" cache_file_path = self._get_cache_file_path(key) with open(cache_file_path, "wb") as cache_file: - resource_data = self._serialize(resource) - cache_file.write(resource_data) + cache_file.write(resource) def _get_cache_file_path(self, key: str) -> Path: key_slug = _get_key_slug(key) @@ -110,34 +99,6 @@ class FileCache(Cache[ResourceType]): return file_path - @abstractmethod - def _serialize(self, resource: ResourceType) -> bytes: - pass - - @abstractmethod - def _deserialize(self, data: bytes) -> ResourceType: - pass - - -class DataCache(FileCache[bytes]): - """Cache of byte array""" - - def _serialize(self, resource: bytes) -> bytes: - return resource - - def _deserialize(self, data: bytes) -> bytes: - return data - - -class ObjectCache(FileCache[ResourceType]): - """Cache pickling objects""" - - def _serialize(self, resource: ResourceType) -> bytes: - return dumps(resource) - - def _deserialize(self, data: bytes) -> ResourceType: - return cast(ResourceType, loads(data)) - def _get_key_slug(url: str) -> str: """Return an unique slug usable as a path name for a given url.""" diff --git a/frontools/config.py b/frontools/config.py index 6db336c..1eaa85c 100644 --- a/frontools/config.py +++ b/frontools/config.py @@ -4,11 +4,13 @@ from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path from re import Pattern from re import compile as re_compile -from typing import Iterable, Optional +from typing import Any, Iterable, Optional, Type from xdg import xdg_config_dirs +from yaml import Loader +from yaml import load as load_yaml -from frontools.cache import Cache, DataCache, NullCache, ObjectCache +from frontools.cache import Cache, FileCache, NullCache from frontools.sources import CachedSource, OverrideSource, Source from frontools.utils import ErrorSummary @@ -49,11 +51,11 @@ class Config: self._default_source_name = default_source_name self._error_summary = ErrorSummary() - remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME) - self._add_source( - REMOTE_SOURCE_NAME, - CachedSource(self._error_summary, self._block_urls, remote_cache), - ) + if use_cache: + remote_cache: Cache = FileCache(REMOTE_SOURCE_NAME) + else: + remote_cache = NullCache() + self._add_source(REMOTE_SOURCE_NAME, CachedSource, remote_cache) self._include_urls = [re_compile(it) for it in include_urls] self._exclude_urls = [re_compile(it) for it in exclude_urls] @@ -106,11 +108,6 @@ class Config: """Return sites configured for this context""" return self._sites.items() - @property - def config_cache(self) -> Cache[object]: - """Get the cache for configuration""" - return self.get_object_cache("config") - def add_site_url(self, name: str, url: str) -> None: """Add an url for a site""" if len(self._include_urls): @@ -126,25 +123,19 @@ class Config: self._sites[name].urls.append(url) + def load_sites_from_yaml(self, yaml_path: str) -> None: + """Load a yaml file containing dictionnary of urls to add as sites.""" + with open(yaml_path, "r", encoding="utf-8") as yaml_file: + yaml_document = load_yaml(yaml_file, Loader) + for site_name, urls in yaml_document.items(): + for url in urls: + self.add_site_url(site_name, url) + def block_url_patterns(self, *patterns: str) -> None: """Will return 500 error for urls matching this pattern.""" for pattern in patterns: self._block_urls.append(re_compile(pattern)) - def get_data_cache(self, name: str) -> Cache[bytes]: - """Get a data cache with the given identifier""" - if self._use_cache: - return DataCache(name) - - return NullCache() - - def get_object_cache(self, name: str) -> Cache[object]: - """Get an object cache with the given identifier""" - if self._use_cache: - return ObjectCache[object](name) - - return NullCache() - def add_override_source( self, name: str, @@ -157,9 +148,7 @@ class Config: next_source = self.default_source else: next_source = self.get_source(next_source_name) - self._sources[name] = OverrideSource( - self._error_summary, self._block_urls, mappings, next_source - ) + self._add_source(name, OverrideSource, mappings, next_source) def get_source(self, name: str) -> Source: """Get an alternate source in the configured ones""" @@ -170,10 +159,14 @@ class Config: def echo_error_summary(self) -> None: self._error_summary.echo() - def _add_source(self, name: str, source: Source) -> None: + def _add_source( + self, name: str, source_class: Type[Source], *args: Any, **kwargs: Any + ) -> None: if name in self._sources: raise Exception(f"Source {name} already configured") - self._sources[name] = source + self._sources[name] = source_class( + self._error_summary, self._block_urls, *args, **kwargs + ) def _find_config() -> Optional[Path]: diff --git a/frontools/sources.py b/frontools/sources.py index 2f3cda6..5bcb464 100644 --- a/frontools/sources.py +++ b/frontools/sources.py @@ -104,7 +104,7 @@ class CachedSource(Source): self, error_summary: ErrorSummary, block_urls: list[Pattern[str]], - cache: Cache[bytes], + cache: Cache, ) -> None: super().__init__(error_summary, block_urls) self._cache = cache diff --git a/noxfile.py b/noxfile.py index 97e8af6..deced1f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -44,7 +44,7 @@ def isort(session: Session) -> None: @nox.session(reuse_venv=True) def mypy(session: Session) -> None: """Run Mypy""" - session.install("mypy", "types-click", "-e", ".") + session.install("mypy", "types-pyyaml", "types-click", "-e", ".") session.run("mypy", "--install-types", *LINT_PATHS) diff --git a/pyproject.toml b/pyproject.toml index 6628fdf..2801295 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ mypy = "^0.942" pylint = "^2.13.2" types-click = "^7.1.8" types-Pillow = "^9.0.11" +types-PyYAML = "^6.0.5" [tool.poetry-dynamic-versioning] enable = true