common: add ability to load urls from nodes

This commit is contained in:
Corentin Sechet 2022-04-12 13:27:45 +02:00
parent f7863d59f4
commit ae22aa4c41
2 changed files with 85 additions and 3 deletions

View File

@ -9,7 +9,7 @@ from typing import Any, Iterable, Optional, Type, Union
from xdg import xdg_config_dirs, xdg_config_home
from frontools.sources import CachedSource, OverrideSource, Source
from frontools.theme_index import ThemeIndex, UrlEntry
from frontools.theme_index import ThemeIndex, UrlEntry, NodeEntry
REMOTE_SOURCE_NAME = "remote"
@ -91,6 +91,10 @@ class Config:
"""Add an url for a theme"""
self._theme_index.add_urls(*urls)
def add_nodes(self, *nodes: NodeEntry) -> None:
"""Add an url for a theme"""
self._theme_index.add_nodes(*nodes)
def add_yaml(self, yaml_path: Union[str, Path]) -> None:
"""Load a yaml file containing dictionnary of urls to add as themes."""
self._theme_index.add_yaml(Path(yaml_path))

View File

@ -1,9 +1,12 @@
"""Store themes and associated urls, providing ways to load them from several sources."""
from asyncio import create_subprocess_shell
from asyncio.subprocess import PIPE
from json import loads
from logging import getLogger
from pathlib import Path
from re import compile as re_compile
from ssl import CERT_NONE, create_default_context
from typing import Iterable, Optional
from typing import AsyncIterable, Iterable, Optional
from aiohttp import ClientSession
from bs4 import BeautifulSoup
@ -17,11 +20,13 @@ _LOGGER = getLogger(__file__)
ThemeIndexData = dict[str, dict[str, list[str]]]
UrlEntry = tuple[str, list[str], Optional[str]] # (url, tags, theme name) tuples
NodeEntry = tuple[str, list[str]] # (server address, tags) tuples
class _Inputs:
urls: list[UrlEntry] = []
yaml_files: list[Path] = []
nodes: list[NodeEntry] = []
class ThemeIndex:
@ -42,6 +47,10 @@ class ThemeIndex:
"""Add an url for a theme"""
self._inputs.urls.extend(urls)
def add_nodes(self, *nodes: NodeEntry) -> None:
"""Add an url for a theme"""
self._inputs.nodes.extend(nodes)
def add_yaml(self, yaml_path: Path) -> None:
"""Load a yaml file containing dictionnary of urls to add as themes."""
self._inputs.yaml_files.append(yaml_path)
@ -52,6 +61,7 @@ class ThemeIndex:
_Inputs.yaml_files.append(index_cache)
else:
await self._load_urls_without_theme()
await self._load_urls_from_nodes()
with open(index_cache, "w") as index_cache_handle:
dump(self._themes, index_cache_handle)
@ -71,11 +81,23 @@ class ThemeIndex:
if theme is None
]
await report_progress(
"Gathering themes from IMIO sites",
"Gathering themes from configured urls",
tasks,
10,
)
async def _load_urls_from_nodes(self) -> None:
async def _load(node: str, node_tags: list[str]) -> None:
async for url, tags, theme in _get_node_urls(node):
assert theme is not None
self._register(url, tags + node_tags, theme)
await report_progress(
"Gathering themes from configured nodes",
[(node, _load(node, tags)) for node, tags in self._inputs.nodes],
5,
)
async def _load_urls_with_theme(self) -> None:
for url, tags, theme in self._inputs.urls:
if theme is not None:
@ -143,3 +165,59 @@ async def _get_theme(url: str) -> Optional[str]:
_LOGGER.error(f"No theme found for url {url}")
return None
CAT_HOBO_JSON_SCRIPT = """
for hobo_json in $(find /var/lib/combo/ -name hobo.json); do
sudo cat $hobo_json
echo ","
done
"""
async def _get_node_urls(
node: str,
) -> AsyncIterable[UrlEntry]:
process = await create_subprocess_shell(
f"ssh {node} -C bash",
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
)
stdout, stderr = await process.communicate(CAT_HOBO_JSON_SCRIPT.encode("utf-8"))
if process.returncode != 0:
print(stderr.decode("utf-8"))
exit(1)
stdout_string = stdout.decode("utf-8").strip()
config_array_content = f"[ {stdout_string[:-1]} ]"
config_array = loads(config_array_content)
for tenant_config in config_array:
services = tenant_config["services"]
variables = tenant_config.get("variables", None)
if variables is None:
continue
theme: Optional[str] = variables.get("theme", None)
if theme is None:
continue
for service in services:
service_id = service.get("service-id", None)
secondary = service.get("secondary", False)
template_name = service.get("template_name", None)
if (
service_id not in ["combo", "wcs", "authentic"]
or secondary
or template_name == "portal-agent"
):
continue
base_url = service.get("base_url", None)
if base_url:
if service_id == "authentic":
base_url = base_url + "login/"
yield (base_url, [service_id], theme)