common: add error_summary accessible in source to report connection errors
This commit is contained in:
parent
3620f4f79e
commit
5bfc487c22
|
@ -1,6 +1,7 @@
|
|||
""" Core module for Frontools"""
|
||||
|
||||
from .config import Config, SiteConfig
|
||||
from .utils import get_page_stylesheets, report_progress
|
||||
from .sources import get_page_stylesheets
|
||||
from .utils import report_progress
|
||||
|
||||
__all__ = ["Config", "SiteConfig", "report_progress", "get_page_stylesheets"]
|
||||
|
|
|
@ -9,7 +9,7 @@ from click import Path as PathArgument
|
|||
from click import argument, group, option, pass_context, pass_obj
|
||||
|
||||
from frontools.cache import Cache
|
||||
from frontools.context import Context
|
||||
from frontools.config import Config
|
||||
from frontools.css import css_diff
|
||||
from frontools.screenshot import screenshot_diff
|
||||
|
||||
|
@ -62,12 +62,12 @@ async def main(
|
|||
exclude_urls: list[str],
|
||||
) -> None:
|
||||
"""Utilities for EO frontend development."""
|
||||
ctx.obj = await Context.load(
|
||||
ctx.obj = await Config.load(
|
||||
config_file, source, not no_cache, include_urls, exclude_urls
|
||||
)
|
||||
|
||||
def _on_close():
|
||||
ctx.obj.print_error_summary()
|
||||
def _on_close() -> None:
|
||||
ctx.obj.echo_error_summary()
|
||||
|
||||
ctx.call_on_close(_on_close)
|
||||
|
||||
|
@ -83,14 +83,14 @@ def prune_caches(cache_names: list[str]) -> None:
|
|||
@argument("right_source", type=str)
|
||||
@pass_obj
|
||||
@_async_command
|
||||
async def css_diff_cli(context: Context, right_source: str) -> None:
|
||||
async def css_diff_cli(config: Config, right_source: str) -> None:
|
||||
"""Diff CSS"""
|
||||
for _, site in context.config.sites:
|
||||
for _, site in config.sites:
|
||||
for site_url in site.urls:
|
||||
await css_diff(
|
||||
site_url,
|
||||
context.config.default_source,
|
||||
context.config.get_source(right_source),
|
||||
config.default_source,
|
||||
config.get_source(right_source),
|
||||
)
|
||||
|
||||
|
||||
|
@ -101,13 +101,13 @@ async def css_diff_cli(context: Context, right_source: str) -> None:
|
|||
@_async_command
|
||||
@pass_obj
|
||||
async def screenshot_diff_cli(
|
||||
context: Context,
|
||||
config: Config,
|
||||
source: str,
|
||||
output_directory: Optional[str],
|
||||
resolution: Optional[str],
|
||||
) -> None:
|
||||
"""Generate screenshot diffs"""
|
||||
await screenshot_diff(context, source, output_directory, resolution=resolution)
|
||||
await screenshot_diff(config, source, output_directory, resolution=resolution)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
@ -9,6 +9,7 @@ from xdg import xdg_config_dirs
|
|||
|
||||
from frontools.cache import Cache, DataCache, NullCache, ObjectCache
|
||||
from frontools.sources import CachedSource, OverrideSource, Source
|
||||
from frontools.utils import ErrorSummary
|
||||
|
||||
REMOTE_SOURCE_NAME = "remote"
|
||||
|
||||
|
@ -45,8 +46,11 @@ class Config:
|
|||
else:
|
||||
self._default_source_name = default_source_name
|
||||
|
||||
self._error_summary = ErrorSummary()
|
||||
remote_cache = self.get_data_cache(REMOTE_SOURCE_NAME)
|
||||
self._add_source(REMOTE_SOURCE_NAME, CachedSource(remote_cache))
|
||||
self._add_source(
|
||||
REMOTE_SOURCE_NAME, CachedSource(self._error_summary, remote_cache)
|
||||
)
|
||||
self._include_urls = [re_compile(it) for it in include_urls]
|
||||
self._exclude_urls = [re_compile(it) for it in exclude_urls]
|
||||
|
||||
|
@ -145,7 +149,7 @@ class Config:
|
|||
next_source = self.default_source
|
||||
else:
|
||||
next_source = self.get_source(next_source_name)
|
||||
self._sources[name] = OverrideSource(mappings, next_source)
|
||||
self._sources[name] = OverrideSource(self._error_summary, mappings, next_source)
|
||||
|
||||
def get_source(self, name: str) -> Source:
|
||||
"""Get an alternate source in the configured ones"""
|
||||
|
@ -153,6 +157,9 @@ class Config:
|
|||
raise Exception(f"No source configured matching {name}")
|
||||
return self._sources[name]
|
||||
|
||||
def echo_error_summary(self) -> None:
|
||||
self._error_summary.echo()
|
||||
|
||||
def _add_source(self, name: str, source: Source) -> None:
|
||||
if name in self._sources:
|
||||
raise Exception(f"Source {name} already configured")
|
||||
|
|
|
@ -1,43 +0,0 @@
|
|||
"""Global context object"""
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from click import echo
|
||||
|
||||
from frontools.config import Config
|
||||
|
||||
|
||||
class Context:
|
||||
"""Configuration object"""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
"""Load config from the given path"""
|
||||
|
||||
self._config = config
|
||||
self._errors: list[str] = []
|
||||
|
||||
@staticmethod
|
||||
async def load(
|
||||
config_path: Optional[Path],
|
||||
default_source_name: Optional[str],
|
||||
use_cache: bool,
|
||||
include_urls: list[str],
|
||||
exclude_urls: list[str],
|
||||
) -> "Context":
|
||||
config = await Config.load(
|
||||
config_path, default_source_name, use_cache, include_urls, exclude_urls
|
||||
)
|
||||
return Context(config)
|
||||
|
||||
def add_error(self, message: str) -> None:
|
||||
self._errors.append(message)
|
||||
|
||||
def print_error_summary(self) -> None:
|
||||
echo("***** Error summary :")
|
||||
for error in self._errors:
|
||||
echo(error, err=True)
|
||||
|
||||
@property
|
||||
def config(self) -> Config:
|
||||
"""Get configuration object for this run"""
|
||||
return self._config
|
|
@ -3,10 +3,9 @@ from pathlib import Path
|
|||
from tempfile import NamedTemporaryFile
|
||||
from typing import Optional
|
||||
|
||||
from aiohttp import ClientConnectionError
|
||||
from PIL import Image, ImageChops
|
||||
|
||||
from frontools.context import Context
|
||||
from frontools.config import Config
|
||||
from frontools.sources import Browser
|
||||
from frontools.utils import (
|
||||
get_default_screenshot_directory,
|
||||
|
@ -16,13 +15,12 @@ from frontools.utils import (
|
|||
|
||||
|
||||
async def screenshot_diff(
|
||||
context: Context,
|
||||
config: Config,
|
||||
right_source_name: str,
|
||||
output_directory: Optional[str],
|
||||
resolution: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Compare pages with or without local css"""
|
||||
config = context.config
|
||||
|
||||
if output_directory is None:
|
||||
output_path = get_default_screenshot_directory()
|
||||
|
@ -56,7 +54,6 @@ async def screenshot_diff(
|
|||
(
|
||||
url,
|
||||
_diff_url(
|
||||
context,
|
||||
left_browser,
|
||||
right_browser,
|
||||
url,
|
||||
|
@ -71,19 +68,14 @@ async def screenshot_diff(
|
|||
|
||||
|
||||
async def _diff_url(
|
||||
context: Context,
|
||||
left: Browser,
|
||||
right: Browser,
|
||||
url: str,
|
||||
output_path: Path,
|
||||
site_name: str,
|
||||
) -> None:
|
||||
try:
|
||||
left_bytes = await _screenshot_url(left, url)
|
||||
right_bytes = await _screenshot_url(right, url)
|
||||
except ClientConnectionError as exception:
|
||||
context.add_error(f'{site_name} : error while loading {url} : {exception}')
|
||||
return
|
||||
left_bytes = await _screenshot_url(left, url)
|
||||
right_bytes = await _screenshot_url(right, url)
|
||||
|
||||
with NamedTemporaryFile(mode="wb") as left_file:
|
||||
left_file.write(left_bytes)
|
||||
|
@ -102,14 +94,10 @@ async def _diff_url(
|
|||
if not output_path.is_dir():
|
||||
output_path.mkdir()
|
||||
|
||||
with open(
|
||||
output_path / f"{site_name}_{url_slug}_left", "wb"
|
||||
) as screenshot_file:
|
||||
with open(output_path / f"{site_name}_{url_slug}_left", "wb") as screenshot_file:
|
||||
screenshot_file.write(left_bytes)
|
||||
|
||||
with open(
|
||||
output_path / f"{site_name}_{url_slug}_right", "wb"
|
||||
) as screenshot_file:
|
||||
with open(output_path / f"{site_name}_{url_slug}_right", "wb") as screenshot_file:
|
||||
screenshot_file.write(right_bytes)
|
||||
|
||||
|
||||
|
|
|
@ -3,16 +3,24 @@ from abc import ABC, abstractmethod
|
|||
from contextlib import asynccontextmanager
|
||||
from re import Pattern
|
||||
from re import compile as re_compile
|
||||
from typing import AsyncGenerator, Optional, cast
|
||||
from typing import AsyncGenerator, AsyncIterable, Optional, cast
|
||||
|
||||
from aiohttp import ClientSession
|
||||
from playwright.async_api import BrowserContext, Route, ViewportSize, async_playwright, Page
|
||||
from bs4 import BeautifulSoup
|
||||
from playwright.async_api import (
|
||||
BrowserContext,
|
||||
Page,
|
||||
Route,
|
||||
ViewportSize,
|
||||
async_playwright,
|
||||
)
|
||||
|
||||
from frontools.cache import Cache
|
||||
from frontools.utils import ErrorSummary
|
||||
|
||||
|
||||
class Browser:
|
||||
def __init__(self, source: 'Source', browser_context: BrowserContext) -> None:
|
||||
def __init__(self, source: "Source", browser_context: BrowserContext) -> None:
|
||||
"""Wraps a browser instance, with helpers methods to load pages."""
|
||||
self._source = source
|
||||
self._browser_context = browser_context
|
||||
|
@ -30,6 +38,9 @@ class Browser:
|
|||
class Source(ABC):
|
||||
"""Base class for sources"""
|
||||
|
||||
def __init__(self, error_summary: ErrorSummary) -> None:
|
||||
self._error_summary = error_summary
|
||||
|
||||
@abstractmethod
|
||||
async def get_url(self, url: str) -> bytes:
|
||||
"""Retrieve the given url content"""
|
||||
|
@ -50,7 +61,9 @@ class Source(ABC):
|
|||
|
||||
async with async_playwright() as pwright:
|
||||
browser = await pwright.firefox.launch(headless=True)
|
||||
context = await browser.new_context(viewport=viewport)
|
||||
context = await browser.new_context(
|
||||
viewport=viewport, ignore_https_errors=True
|
||||
)
|
||||
yield Browser(self, context)
|
||||
await browser.close()
|
||||
|
||||
|
@ -62,7 +75,8 @@ class Source(ABC):
|
|||
class CachedSource(Source):
|
||||
"""Source loading urls from the internet."""
|
||||
|
||||
def __init__(self, cache: Cache[bytes]) -> None:
|
||||
def __init__(self, error_summary: ErrorSummary, cache: Cache[bytes]) -> None:
|
||||
super().__init__(error_summary)
|
||||
self._cache = cache
|
||||
|
||||
async def get_url(self, url: str) -> bytes:
|
||||
|
@ -81,9 +95,11 @@ class OverrideSource(Source):
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
error_summary: ErrorSummary,
|
||||
mappings: list[tuple[str, str]],
|
||||
next_source: Source,
|
||||
):
|
||||
super().__init__(error_summary)
|
||||
self._mappings: list[tuple[Pattern[str], str]] = []
|
||||
self._next_source = next_source
|
||||
|
||||
|
@ -100,3 +116,15 @@ class OverrideSource(Source):
|
|||
return mapped_file.read()
|
||||
|
||||
return await self._next_source.get_url(url)
|
||||
|
||||
|
||||
async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]:
|
||||
"""Return styleheets urls for a given page."""
|
||||
page_content = await source.get_url(url)
|
||||
page_html = BeautifulSoup(page_content, features="html5lib")
|
||||
links = page_html.find_all("link")
|
||||
for link in links:
|
||||
if "stylesheet" not in link.get("rel", []):
|
||||
continue
|
||||
|
||||
yield link["href"]
|
||||
|
|
|
@ -4,13 +4,25 @@ from datetime import datetime
|
|||
from os.path import expandvars
|
||||
from pathlib import Path
|
||||
from re import compile as re_compile
|
||||
from typing import AsyncIterable, Awaitable, cast
|
||||
from typing import Awaitable, cast
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from click import progressbar
|
||||
from click import echo, progressbar
|
||||
from xdg import xdg_config_home
|
||||
|
||||
from frontools.sources import Source
|
||||
|
||||
class ErrorSummary:
|
||||
def __init__(self) -> None:
|
||||
self._errors: list[str] = []
|
||||
|
||||
def add_error(self, message: str) -> None:
|
||||
self._errors.append(message)
|
||||
|
||||
def echo(self) -> None:
|
||||
if not len(self._errors):
|
||||
return
|
||||
echo("***** Error summary :")
|
||||
for error in self._errors:
|
||||
echo(error, err=True)
|
||||
|
||||
|
||||
def get_default_screenshot_directory() -> Path:
|
||||
|
@ -39,7 +51,9 @@ def get_default_screenshot_directory() -> Path:
|
|||
TaskListType = list[tuple[str, Awaitable[None]]]
|
||||
|
||||
|
||||
async def report_progress(label: str, task_list: TaskListType, nb_workers: int = 5) -> None:
|
||||
async def report_progress(
|
||||
label: str, task_list: TaskListType, nb_workers: int = 5
|
||||
) -> None:
|
||||
"""Show a progress bar for a long running task list"""
|
||||
iterator = iter(task_list)
|
||||
|
||||
|
@ -62,18 +76,6 @@ async def report_progress(label: str, task_list: TaskListType, nb_workers: int =
|
|||
await gather(*workers)
|
||||
|
||||
|
||||
async def get_page_stylesheets(source: Source, url: str) -> AsyncIterable[str]:
|
||||
"""Return styleheets urls for a given page."""
|
||||
page_content = await source.get_url(url)
|
||||
page_html = BeautifulSoup(page_content, features="html5lib")
|
||||
links = page_html.find_all("link")
|
||||
for link in links:
|
||||
if "stylesheet" not in link.get("rel", []):
|
||||
continue
|
||||
|
||||
yield link["href"]
|
||||
|
||||
|
||||
def get_url_slug(url: str) -> str:
|
||||
"""Return an unique slug usable as a path name for a given url."""
|
||||
return url.replace("_", "___").replace("/", "__").replace(":", "_")
|
||||
|
|
Loading…
Reference in New Issue