diff --git a/frontools/__init__.py b/frontools/__init__.py index f460404..710b708 100644 --- a/frontools/__init__.py +++ b/frontools/__init__.py @@ -1,6 +1,7 @@ """ Core module for Frontools""" from .config import Config, SiteConfig -from .utils import get_page_stylesheets, report_progress +from .sources import get_page_stylesheets +from .utils import report_progress __all__ = ["Config", "SiteConfig", "report_progress", "get_page_stylesheets"] diff --git a/frontools/cli.py b/frontools/cli.py index acbe581..1e489dd 100644 --- a/frontools/cli.py +++ b/frontools/cli.py @@ -9,7 +9,7 @@ from click import Path as PathArgument from click import argument, group, option, pass_context, pass_obj from frontools.cache import Cache -from frontools.context import Context +from frontools.config import Config from frontools.css import css_diff from frontools.screenshot import screenshot_diff @@ -62,12 +62,12 @@ async def main( exclude_urls: list[str], ) -> None: """Utilities for EO frontend development.""" - ctx.obj = await Context.load( + ctx.obj = await Config.load( config_file, source, not no_cache, include_urls, exclude_urls ) - def _on_close(): - ctx.obj.print_error_summary() + def _on_close() -> None: + ctx.obj.echo_error_summary() ctx.call_on_close(_on_close) @@ -83,14 +83,14 @@ def prune_caches(cache_names: list[str]) -> None: @argument("right_source", type=str) @pass_obj @_async_command -async def css_diff_cli(context: Context, right_source: str) -> None: +async def css_diff_cli(config: Config, right_source: str) -> None: """Diff CSS""" - for _, site in context.config.sites: + for _, site in config.sites: for site_url in site.urls: await css_diff( site_url, - context.config.default_source, - context.config.get_source(right_source), + config.default_source, + config.get_source(right_source), ) @@ -101,13 +101,13 @@ async def css_diff_cli(context: Context, right_source: str) -> None: @_async_command @pass_obj async def screenshot_diff_cli( - context: Context, + config: Config, source: str, output_directory: Optional[str], resolution: Optional[str], ) -> None: """Generate screenshot diffs""" - await screenshot_diff(context, source, output_directory, resolution=resolution) + await screenshot_diff(config, source, output_directory, resolution=resolution) if __name__ == "__main__": diff --git a/frontools/config.py b/frontools/config.py index 0bd3a81..6a0b7bf 100644 --- a/frontools/config.py +++ b/frontools/config.py @@ -9,6 +9,7 @@ from xdg import xdg_config_dirs from frontools.cache import Cache, DataCache, NullCache, ObjectCache from frontools.sources import CachedSource, OverrideSource, Source +from frontools.utils import ErrorSummary REMOTE_SOURCE_NAME = "remote" @@ -45,8 +46,11 @@ class Config: else: self._default_source_name = default_source_name + self._error_summary = ErrorSummary() remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME) - self._add_source(REMOTE_SOURCE_NAME, CachedSource(remote_cache)) + self._add_source( + REMOTE_SOURCE_NAME, CachedSource(self._error_summary, remote_cache) + ) self._include_urls = [re_compile(it) for it in include_urls] self._exclude_urls = [re_compile(it) for it in exclude_urls] @@ -145,7 +149,7 @@ class Config: next_source = self.default_source else: next_source = self.get_source(next_source_name) - self._sources[name] = OverrideSource(mappings, next_source) + self._sources[name] = OverrideSource(self._error_summary, mappings, next_source) def get_source(self, name: str) -> Source: """Get an alternate source in the configured ones""" @@ -153,6 +157,9 @@ class Config: raise Exception(f"No source configured matching {name}") return self._sources[name] + def echo_error_summary(self) -> None: + self._error_summary.echo() + def _add_source(self, name: str, source: Source) -> None: if name in self._sources: raise Exception(f"Source {name} already configured") diff --git a/frontools/context.py b/frontools/context.py deleted file mode 100644 index 9c72e38..0000000 --- a/frontools/context.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Global context object""" -from pathlib import Path -from typing import Optional - -from click import echo - -from frontools.config import Config - - -class Context: - """Configuration object""" - - def __init__(self, config: Config) -> None: - """Load config from the given path""" - - self._config = config - self._errors: list[str] = [] - - @staticmethod - async def load( - config_path: Optional[Path], - default_source_name: Optional[str], - use_cache: bool, - include_urls: list[str], - exclude_urls: list[str], - ) -> "Context": - config = await Config.load( - config_path, default_source_name, use_cache, include_urls, exclude_urls - ) - return Context(config) - - def add_error(self, message: str) -> None: - self._errors.append(message) - - def print_error_summary(self) -> None: - echo("***** Error summary :") - for error in self._errors: - echo(error, err=True) - - @property - def config(self) -> Config: - """Get configuration object for this run""" - return self._config diff --git a/frontools/screenshot.py b/frontools/screenshot.py index d2ac615..033a941 100644 --- a/frontools/screenshot.py +++ b/frontools/screenshot.py @@ -3,10 +3,9 @@ from pathlib import Path from tempfile import NamedTemporaryFile from typing import Optional -from aiohttp import ClientConnectionError from PIL import Image, ImageChops -from frontools.context import Context +from frontools.config import Config from frontools.sources import Browser from frontools.utils import ( get_default_screenshot_directory, @@ -16,13 +15,12 @@ from frontools.utils import ( async def screenshot_diff( - context: Context, + config: Config, right_source_name: str, output_directory: Optional[str], resolution: Optional[str] = None, ) -> None: """Compare pages with or without local css""" - config = context.config if output_directory is None: output_path = get_default_screenshot_directory() @@ -56,7 +54,6 @@ async def screenshot_diff( ( url, _diff_url( - context, left_browser, right_browser, url, @@ -71,19 +68,14 @@ async def screenshot_diff( async def _diff_url( - context: Context, left: Browser, right: Browser, url: str, output_path: Path, site_name: str, ) -> None: - try: - left_bytes = await _screenshot_url(left, url) - right_bytes = await _screenshot_url(right, url) - except ClientConnectionError as exception: - context.add_error(f'{site_name} : error while loading {url} : {exception}') - return + left_bytes = await _screenshot_url(left, url) + right_bytes = await _screenshot_url(right, url) with NamedTemporaryFile(mode="wb") as left_file: left_file.write(left_bytes) @@ -102,14 +94,10 @@ async def _diff_url( if not output_path.is_dir(): output_path.mkdir() - with open( - output_path / f"{site_name}_{url_slug}_left", "wb" - ) as screenshot_file: + with open(output_path / f"{site_name}_{url_slug}_left", "wb") as screenshot_file: screenshot_file.write(left_bytes) - with open( - output_path / f"{site_name}_{url_slug}_right", "wb" - ) as screenshot_file: + with open(output_path / f"{site_name}_{url_slug}_right", "wb") as screenshot_file: screenshot_file.write(right_bytes) diff --git a/frontools/sources.py b/frontools/sources.py index c434ded..1b34ca9 100644 --- a/frontools/sources.py +++ b/frontools/sources.py @@ -3,16 +3,24 @@ from abc import ABC, abstractmethod from contextlib import asynccontextmanager from re import Pattern from re import compile as re_compile -from typing import AsyncGenerator, Optional, cast +from typing import AsyncGenerator, AsyncIterable, Optional, cast from aiohttp import ClientSession -from playwright.async_api import BrowserContext, Route, ViewportSize, async_playwright, Page +from bs4 import BeautifulSoup +from playwright.async_api import ( + BrowserContext, + Page, + Route, + ViewportSize, + async_playwright, +) from frontools.cache import Cache +from frontools.utils import ErrorSummary class Browser: - def __init__(self, source: 'Source', browser_context: BrowserContext) -> None: + def __init__(self, source: "Source", browser_context: BrowserContext) -> None: """Wraps a browser instance, with helpers methods to load pages.""" self._source = source self._browser_context = browser_context @@ -30,6 +38,9 @@ class Browser: class Source(ABC): """Base class for sources""" + def __init__(self, error_summary: ErrorSummary) -> None: + self._error_summary = error_summary + @abstractmethod async def get_url(self, url: str) -> bytes: """Retrieve the given url content""" @@ -50,7 +61,9 @@ class Source(ABC): async with async_playwright() as pwright: browser = await pwright.firefox.launch(headless=True) - context = await browser.new_context(viewport=viewport) + context = await browser.new_context( + viewport=viewport, ignore_https_errors=True + ) yield Browser(self, context) await browser.close() @@ -62,7 +75,8 @@ class Source(ABC): class CachedSource(Source): """Source loading urls from the internet.""" - def __init__(self, cache: Cache[bytes]) -> None: + def __init__(self, error_summary: ErrorSummary, cache: Cache[bytes]) -> None: + super().__init__(error_summary) self._cache = cache async def get_url(self, url: str) -> bytes: @@ -81,9 +95,11 @@ class OverrideSource(Source): def __init__( self, + error_summary: ErrorSummary, mappings: list[tuple[str, str]], next_source: Source, ): + super().__init__(error_summary) self._mappings: list[tuple[Pattern[str], str]] = [] self._next_source = next_source @@ -100,3 +116,15 @@ class OverrideSource(Source): return mapped_file.read() return await self._next_source.get_url(url) + + +async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]: + """Return styleheets urls for a given page.""" + page_content = await source.get_url(url) + page_html = BeautifulSoup(page_content, features="html5lib") + links = page_html.find_all("link") + for link in links: + if "stylesheet" not in link.get("rel", []): + continue + + yield link["href"] diff --git a/frontools/utils.py b/frontools/utils.py index 8883527..e143c19 100644 --- a/frontools/utils.py +++ b/frontools/utils.py @@ -4,13 +4,25 @@ from datetime import datetime from os.path import expandvars from pathlib import Path from re import compile as re_compile -from typing import AsyncIterable, Awaitable, cast +from typing import Awaitable, cast -from bs4 import BeautifulSoup -from click import progressbar +from click import echo, progressbar from xdg import xdg_config_home -from frontools.sources import Source + +class ErrorSummary: + def __init__(self) -> None: + self._errors: list[str] = [] + + def add_error(self, message: str) -> None: + self._errors.append(message) + + def echo(self) -> None: + if not len(self._errors): + return + echo("***** Error summary :") + for error in self._errors: + echo(error, err=True) def get_default_screenshot_directory() -> Path: @@ -39,7 +51,9 @@ def get_default_screenshot_directory() -> Path: TaskListType = list[tuple[str, Awaitable[None]]] -async def report_progress(label: str, task_list: TaskListType, nb_workers: int = 5) -> None: +async def report_progress( + label: str, task_list: TaskListType, nb_workers: int = 5 +) -> None: """Show a progress bar for a long running task list""" iterator = iter(task_list) @@ -62,18 +76,6 @@ async def report_progress(label: str, task_list: TaskListType, nb_workers: int = await gather(*workers) -async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]: - """Return styleheets urls for a given page.""" - page_content = await source.get_url(url) - page_html = BeautifulSoup(page_content, features="html5lib") - links = page_html.find_all("link") - for link in links: - if "stylesheet" not in link.get("rel", []): - continue - - yield link["href"] - - def get_url_slug(url: str) -> str: """Return an unique slug usable as a path name for a given url.""" return url.replace("_", "___").replace("/", "__").replace(":", "_")