common: add error_summary accessible in source to report connection errors

This commit is contained in:
Corentin Sechet 2022-04-11 14:29:50 +02:00
parent 3620f4f79e
commit 5bfc487c22
7 changed files with 79 additions and 96 deletions

View File

@ -1,6 +1,7 @@
""" Core module for Frontools"""
from .config import Config, SiteConfig
from .utils import get_page_stylesheets, report_progress
from .sources import get_page_stylesheets
from .utils import report_progress
__all__ = ["Config", "SiteConfig", "report_progress", "get_page_stylesheets"]

View File

@ -9,7 +9,7 @@ from click import Path as PathArgument
from click import argument, group, option, pass_context, pass_obj
from frontools.cache import Cache
from frontools.context import Context
from frontools.config import Config
from frontools.css import css_diff
from frontools.screenshot import screenshot_diff
@ -62,12 +62,12 @@ async def main(
exclude_urls: list[str],
) -> None:
"""Utilities for EO frontend development."""
ctx.obj = await Context.load(
ctx.obj = await Config.load(
config_file, source, not no_cache, include_urls, exclude_urls
)
def _on_close():
ctx.obj.print_error_summary()
def _on_close() -> None:
ctx.obj.echo_error_summary()
ctx.call_on_close(_on_close)
@ -83,14 +83,14 @@ def prune_caches(cache_names: list[str]) -> None:
@argument("right_source", type=str)
@pass_obj
@_async_command
async def css_diff_cli(context: Context, right_source: str) -> None:
async def css_diff_cli(config: Config, right_source: str) -> None:
"""Diff CSS"""
for _, site in context.config.sites:
for _, site in config.sites:
for site_url in site.urls:
await css_diff(
site_url,
context.config.default_source,
context.config.get_source(right_source),
config.default_source,
config.get_source(right_source),
)
@ -101,13 +101,13 @@ async def css_diff_cli(context: Context, right_source: str) -> None:
@_async_command
@pass_obj
async def screenshot_diff_cli(
context: Context,
config: Config,
source: str,
output_directory: Optional[str],
resolution: Optional[str],
) -> None:
"""Generate screenshot diffs"""
await screenshot_diff(context, source, output_directory, resolution=resolution)
await screenshot_diff(config, source, output_directory, resolution=resolution)
if __name__ == "__main__":

View File

@ -9,6 +9,7 @@ from xdg import xdg_config_dirs
from frontools.cache import Cache, DataCache, NullCache, ObjectCache
from frontools.sources import CachedSource, OverrideSource, Source
from frontools.utils import ErrorSummary
REMOTE_SOURCE_NAME = "remote"
@ -45,8 +46,11 @@ class Config:
else:
self._default_source_name = default_source_name
self._error_summary = ErrorSummary()
remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME)
self._add_source(REMOTE_SOURCE_NAME, CachedSource(remote_cache))
self._add_source(
REMOTE_SOURCE_NAME, CachedSource(self._error_summary, remote_cache)
)
self._include_urls = [re_compile(it) for it in include_urls]
self._exclude_urls = [re_compile(it) for it in exclude_urls]
@ -145,7 +149,7 @@ class Config:
next_source = self.default_source
else:
next_source = self.get_source(next_source_name)
self._sources[name] = OverrideSource(mappings, next_source)
self._sources[name] = OverrideSource(self._error_summary, mappings, next_source)
def get_source(self, name: str) -> Source:
"""Get an alternate source in the configured ones"""
@ -153,6 +157,9 @@ class Config:
raise Exception(f"No source configured matching {name}")
return self._sources[name]
def echo_error_summary(self) -> None:
self._error_summary.echo()
def _add_source(self, name: str, source: Source) -> None:
if name in self._sources:
raise Exception(f"Source {name} already configured")

View File

@ -1,43 +0,0 @@
"""Global context object"""
from pathlib import Path
from typing import Optional
from click import echo
from frontools.config import Config
class Context:
"""Configuration object"""
def __init__(self, config: Config) -> None:
"""Load config from the given path"""
self._config = config
self._errors: list[str] = []
@staticmethod
async def load(
config_path: Optional[Path],
default_source_name: Optional[str],
use_cache: bool,
include_urls: list[str],
exclude_urls: list[str],
) -> "Context":
config = await Config.load(
config_path, default_source_name, use_cache, include_urls, exclude_urls
)
return Context(config)
def add_error(self, message: str) -> None:
self._errors.append(message)
def print_error_summary(self) -> None:
echo("***** Error summary :")
for error in self._errors:
echo(error, err=True)
@property
def config(self) -> Config:
"""Get configuration object for this run"""
return self._config

View File

@ -3,10 +3,9 @@ from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional
from aiohttp import ClientConnectionError
from PIL import Image, ImageChops
from frontools.context import Context
from frontools.config import Config
from frontools.sources import Browser
from frontools.utils import (
get_default_screenshot_directory,
@ -16,13 +15,12 @@ from frontools.utils import (
async def screenshot_diff(
context: Context,
config: Config,
right_source_name: str,
output_directory: Optional[str],
resolution: Optional[str] = None,
) -> None:
"""Compare pages with or without local css"""
config = context.config
if output_directory is None:
output_path = get_default_screenshot_directory()
@ -56,7 +54,6 @@ async def screenshot_diff(
(
url,
_diff_url(
context,
left_browser,
right_browser,
url,
@ -71,19 +68,14 @@ async def screenshot_diff(
async def _diff_url(
context: Context,
left: Browser,
right: Browser,
url: str,
output_path: Path,
site_name: str,
) -> None:
try:
left_bytes = await _screenshot_url(left, url)
right_bytes = await _screenshot_url(right, url)
except ClientConnectionError as exception:
context.add_error(f'{site_name} : error while loading {url} : {exception}')
return
left_bytes = await _screenshot_url(left, url)
right_bytes = await _screenshot_url(right, url)
with NamedTemporaryFile(mode="wb") as left_file:
left_file.write(left_bytes)
@ -102,14 +94,10 @@ async def _diff_url(
if not output_path.is_dir():
output_path.mkdir()
with open(
output_path / f"{site_name}_{url_slug}_left", "wb"
) as screenshot_file:
with open(output_path / f"{site_name}_{url_slug}_left", "wb") as screenshot_file:
screenshot_file.write(left_bytes)
with open(
output_path / f"{site_name}_{url_slug}_right", "wb"
) as screenshot_file:
with open(output_path / f"{site_name}_{url_slug}_right", "wb") as screenshot_file:
screenshot_file.write(right_bytes)

View File

@ -3,16 +3,24 @@ from abc import ABC, abstractmethod
from contextlib import asynccontextmanager
from re import Pattern
from re import compile as re_compile
from typing import AsyncGenerator, Optional, cast
from typing import AsyncGenerator, AsyncIterable, Optional, cast
from aiohttp import ClientSession
from playwright.async_api import BrowserContext, Route, ViewportSize, async_playwright, Page
from bs4 import BeautifulSoup
from playwright.async_api import (
BrowserContext,
Page,
Route,
ViewportSize,
async_playwright,
)
from frontools.cache import Cache
from frontools.utils import ErrorSummary
class Browser:
def __init__(self, source: 'Source', browser_context: BrowserContext) -> None:
def __init__(self, source: "Source", browser_context: BrowserContext) -> None:
"""Wraps a browser instance, with helpers methods to load pages."""
self._source = source
self._browser_context = browser_context
@ -30,6 +38,9 @@ class Browser:
class Source(ABC):
"""Base class for sources"""
def __init__(self, error_summary: ErrorSummary) -> None:
self._error_summary = error_summary
@abstractmethod
async def get_url(self, url: str) -> bytes:
"""Retrieve the given url content"""
@ -50,7 +61,9 @@ class Source(ABC):
async with async_playwright() as pwright:
browser = await pwright.firefox.launch(headless=True)
context = await browser.new_context(viewport=viewport)
context = await browser.new_context(
viewport=viewport, ignore_https_errors=True
)
yield Browser(self, context)
await browser.close()
@ -62,7 +75,8 @@ class Source(ABC):
class CachedSource(Source):
"""Source loading urls from the internet."""
def __init__(self, cache: Cache[bytes]) -> None:
def __init__(self, error_summary: ErrorSummary, cache: Cache[bytes]) -> None:
super().__init__(error_summary)
self._cache = cache
async def get_url(self, url: str) -> bytes:
@ -81,9 +95,11 @@ class OverrideSource(Source):
def __init__(
self,
error_summary: ErrorSummary,
mappings: list[tuple[str, str]],
next_source: Source,
):
super().__init__(error_summary)
self._mappings: list[tuple[Pattern[str], str]] = []
self._next_source = next_source
@ -100,3 +116,15 @@ class OverrideSource(Source):
return mapped_file.read()
return await self._next_source.get_url(url)
async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]:
"""Return styleheets urls for a given page."""
page_content = await source.get_url(url)
page_html = BeautifulSoup(page_content, features="html5lib")
links = page_html.find_all("link")
for link in links:
if "stylesheet" not in link.get("rel", []):
continue
yield link["href"]

View File

@ -4,13 +4,25 @@ from datetime import datetime
from os.path import expandvars
from pathlib import Path
from re import compile as re_compile
from typing import AsyncIterable, Awaitable, cast
from typing import Awaitable, cast
from bs4 import BeautifulSoup
from click import progressbar
from click import echo, progressbar
from xdg import xdg_config_home
from frontools.sources import Source
class ErrorSummary:
def __init__(self) -> None:
self._errors: list[str] = []
def add_error(self, message: str) -> None:
self._errors.append(message)
def echo(self) -> None:
if not len(self._errors):
return
echo("***** Error summary :")
for error in self._errors:
echo(error, err=True)
def get_default_screenshot_directory() -> Path:
@ -39,7 +51,9 @@ def get_default_screenshot_directory() -> Path:
TaskListType = list[tuple[str, Awaitable[None]]]
async def report_progress(label: str, task_list: TaskListType, nb_workers: int = 5) -> None:
async def report_progress(
label: str, task_list: TaskListType, nb_workers: int = 5
) -> None:
"""Show a progress bar for a long running task list"""
iterator = iter(task_list)
@ -62,18 +76,6 @@ async def report_progress(label: str, task_list: TaskListType, nb_workers: int =
await gather(*workers)
async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]:
"""Return styleheets urls for a given page."""
page_content = await source.get_url(url)
page_html = BeautifulSoup(page_content, features="html5lib")
links = page_html.find_all("link")
for link in links:
if "stylesheet" not in link.get("rel", []):
continue
yield link["href"]
def get_url_slug(url: str) -> str:
"""Return an unique slug usable as a path name for a given url."""
return url.replace("_", "___").replace("/", "__").replace(":", "_")