entrouvert
/
misc-csechet
17
0
Fork 0
Code Issues Pull Requests Releases Activity

common: add support for filtering url by tags

This commit is contained in:
Corentin Sechet 2022-04-12 00:58:01 +02:00
parent ccade0bf6e
commit 4f9dd27045
4 changed files with 96 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
frontools/cli.py
View File

@ -50,7 +50,21 @@ def _async_command(function: Any) -> Any:
"--exclude-urls",
type=str,
multiple=True,
help="Patterns of urls to ignore",
help="Ignore urls matching this pattern",
)
@option(
"-i",
"--include-tags",
type=str,
multiple=True,
help="Take into account only urls with those tags",
)
@option(
"-e",
"--exclude-tags",
type=str,
multiple=True,
help="Ignore urls matching with those tags",
)
@_async_command
async def main(
@ -60,10 +74,18 @@ async def main(
no_cache: bool,
include_urls: list[str],
exclude_urls: list[str],
include_tags: list[str],
exclude_tags: list[str],
) -> None:
"""Utilities for EO frontend development."""
ctx.obj = await Config.load(
config_file, source, not no_cache, include_urls, exclude_urls
config_file,
source,
not no_cache,
include_urls,
exclude_urls,
include_tags,
exclude_tags,
)
def _on_close() -> None:
@ -85,13 +107,12 @@ def prune_caches(cache_names: list[str]) -> None:
@_async_command
async def css_diff_cli(config: Config, right_source: str) -> None:
"""Diff CSS"""
for _, site in config.sites:
for site_url in site.urls:
await css_diff(
site_url,
config.default_source,
config.get_source(right_source),
)
for _, url in config.urls:
await css_diff(
url,
config.default_source,
config.get_source(right_source),
)
@main.command(name="screenshot-diff")

83
frontools/config.py
View File

@ -21,13 +21,18 @@ class ConfigError(Exception):
"""Error raised on config error"""
class UrlConfig:
"""Config for an url"""
def __init__(self) -> None:
self.tags: set[str] = set()
class SiteConfig:
"""Configuration object for a particular website"""
urls: list[str]
def __init__(self, urls: Iterable[str]):
self.urls = list(urls)
def __init__(self) -> None:
self.urls: dict[str, UrlConfig] = {}
class Config:
@ -39,6 +44,8 @@ class Config:
default_source_name: Optional[str],
include_urls: list[str],
exclude_urls: list[str],
include_tags: list[str],
exclude_tags: list[str],
):
self._use_cache = use_cache
self._sources: dict[str, Source] = {}
@ -58,6 +65,8 @@ class Config:
self._add_source(REMOTE_SOURCE_NAME, CachedSource, remote_cache)
self._include_urls = [re_compile(it) for it in include_urls]
self._exclude_urls = [re_compile(it) for it in exclude_urls]
self._include_tags = set(include_tags)
self._exclude_tags = set(exclude_tags)
@staticmethod
async def load(
@ -66,9 +75,18 @@ class Config:
use_cache: bool,
include_urls: list[str],
exclude_urls: list[str],
include_tags: list[str],
exclude_tags: list[str],
) -> "Config":
"""Load config from the given path"""
config = Config(use_cache, default_source_name, include_urls, exclude_urls)
config = Config(
use_cache,
default_source_name,
include_urls,
exclude_urls,
include_tags,
exclude_tags,
)
if config_path is None:
config_path = _find_config()
@ -104,32 +122,42 @@ class Config:
return self.get_source(self._default_source_name)
@property
def sites(self) -> Iterable[tuple[str, SiteConfig]]:
def urls(self) -> Iterable[tuple[str, str]]:
"""Return sites configured for this context"""
return self._sites.items()
for site_name, site in self._sites.items():
for url, config in site.urls.items():
if self._filter(url, config.tags):
yield site_name, url
def add_site_url(self, name: str, url: str) -> None:
def add_site_url(
self, name: str, url: str, tags: Optional[Iterable[str]] = None
) -> None:
"""Add an url for a site"""
if len(self._include_urls):
if all([not it.match(url) for it in self._include_urls]):
return
if len(self._exclude_urls):
if any([it.match(url) for it in self._exclude_urls]):
return
site = self._sites.get(name, None)
if site is None:
site = SiteConfig()
self._sites[name] = site
if name not in self._sites:
self._sites[name] = SiteConfig([])
if tags is None:
new_tags = set()
else:
new_tags = set(tags)
self._sites[name].urls.append(url)
url_config = site.urls.get(url, None)
if url_config is None:
url_config = UrlConfig()
site.urls[url] = url_config
url_config.tags.update(new_tags)
def load_sites_from_yaml(self, yaml_path: str) -> None:
"""Load a yaml file containing dictionnary of urls to add as sites."""
with open(yaml_path, "r", encoding="utf-8") as yaml_file:
yaml_document = load_yaml(yaml_file, Loader)
for site_name, urls in yaml_document.items():
for url in urls:
self.add_site_url(site_name, url)
for url, tags in urls.items():
self.add_site_url(site_name, url, tags)
def block_url_patterns(self, *patterns: str) -> None:
"""Will return 500 error for urls matching this pattern."""
@ -168,6 +196,23 @@ class Config:
self._error_summary, self._block_urls, *args, **kwargs
)
def _filter(self, url: str, tags: set[str]) -> bool:
if self._include_urls:
if all([not it.match(url) for it in self._include_urls]):
return False
if self._exclude_urls:
if any([it.match(url) for it in self._exclude_urls]):
return False
if self._include_tags and not self._include_tags & tags:
return False
if self._exclude_tags and self._exclude_tags & tags:
return False
return True
def _find_config() -> Optional[Path]:
local_config = Path(".frontools.py")

8
frontools/screenshot.py
View File

@ -34,12 +34,6 @@ async def screenshot_diff(
async with left_source.get_browser(width=screen_width) as left_browser:
async with right_source.get_browser(width=screen_width) as right_browser:
urls = [
(site_name, url)
for (site_name, site) in config.sites
for url in site.urls
]
await report_progress(
"Screenshoting",
[
@ -53,7 +47,7 @@ async def screenshot_diff(
site_name,
),
)
for (site_name, url) in urls
for (site_name, url) in config.urls
],
nb_workers=3,
)

2
frontools/sources.py
View File

@ -29,7 +29,7 @@ class Browser:
self._browser_context = browser_context
@asynccontextmanager
async def load_page(self, url: str) -> AsyncGenerator[Optional[Page], None]:
async def load_page(self, url: str) -> AsyncGenerator[Page, None]:
page = await self._browser_context.new_page()
await page.route("*", self._source.route)
for retry in range(0, 3):