common: add ability to load urls from nodes
This commit is contained in:
parent
f7863d59f4
commit
ae22aa4c41
|
@ -9,7 +9,7 @@ from typing import Any, Iterable, Optional, Type, Union
|
|||
from xdg import xdg_config_dirs, xdg_config_home
|
||||
|
||||
from frontools.sources import CachedSource, OverrideSource, Source
|
||||
from frontools.theme_index import ThemeIndex, UrlEntry
|
||||
from frontools.theme_index import ThemeIndex, UrlEntry, NodeEntry
|
||||
|
||||
REMOTE_SOURCE_NAME = "remote"
|
||||
|
||||
|
@ -91,6 +91,10 @@ class Config:
|
|||
"""Add an url for a theme"""
|
||||
self._theme_index.add_urls(*urls)
|
||||
|
||||
def add_nodes(self, *nodes: NodeEntry) -> None:
|
||||
"""Add an url for a theme"""
|
||||
self._theme_index.add_nodes(*nodes)
|
||||
|
||||
def add_yaml(self, yaml_path: Union[str, Path]) -> None:
|
||||
"""Load a yaml file containing dictionnary of urls to add as themes."""
|
||||
self._theme_index.add_yaml(Path(yaml_path))
|
||||
|
|
|
@ -1,9 +1,12 @@
|
|||
"""Store themes and associated urls, providing ways to load them from several sources."""
|
||||
from asyncio import create_subprocess_shell
|
||||
from asyncio.subprocess import PIPE
|
||||
from json import loads
|
||||
from logging import getLogger
|
||||
from pathlib import Path
|
||||
from re import compile as re_compile
|
||||
from ssl import CERT_NONE, create_default_context
|
||||
from typing import Iterable, Optional
|
||||
from typing import AsyncIterable, Iterable, Optional
|
||||
|
||||
from aiohttp import ClientSession
|
||||
from bs4 import BeautifulSoup
|
||||
|
@ -17,11 +20,13 @@ _LOGGER = getLogger(__file__)
|
|||
|
||||
ThemeIndexData = dict[str, dict[str, list[str]]]
|
||||
UrlEntry = tuple[str, list[str], Optional[str]] # (url, tags, theme name) tuples
|
||||
NodeEntry = tuple[str, list[str]] # (server address, tags) tuples
|
||||
|
||||
|
||||
class _Inputs:
|
||||
urls: list[UrlEntry] = []
|
||||
yaml_files: list[Path] = []
|
||||
nodes: list[NodeEntry] = []
|
||||
|
||||
|
||||
class ThemeIndex:
|
||||
|
@ -42,6 +47,10 @@ class ThemeIndex:
|
|||
"""Add an url for a theme"""
|
||||
self._inputs.urls.extend(urls)
|
||||
|
||||
def add_nodes(self, *nodes: NodeEntry) -> None:
|
||||
"""Add an url for a theme"""
|
||||
self._inputs.nodes.extend(nodes)
|
||||
|
||||
def add_yaml(self, yaml_path: Path) -> None:
|
||||
"""Load a yaml file containing dictionnary of urls to add as themes."""
|
||||
self._inputs.yaml_files.append(yaml_path)
|
||||
|
@ -52,6 +61,7 @@ class ThemeIndex:
|
|||
_Inputs.yaml_files.append(index_cache)
|
||||
else:
|
||||
await self._load_urls_without_theme()
|
||||
await self._load_urls_from_nodes()
|
||||
with open(index_cache, "w") as index_cache_handle:
|
||||
dump(self._themes, index_cache_handle)
|
||||
|
||||
|
@ -71,11 +81,23 @@ class ThemeIndex:
|
|||
if theme is None
|
||||
]
|
||||
await report_progress(
|
||||
"Gathering themes from IMIO sites",
|
||||
"Gathering themes from configured urls",
|
||||
tasks,
|
||||
10,
|
||||
)
|
||||
|
||||
async def _load_urls_from_nodes(self) -> None:
|
||||
async def _load(node: str, node_tags: list[str]) -> None:
|
||||
async for url, tags, theme in _get_node_urls(node):
|
||||
assert theme is not None
|
||||
self._register(url, tags + node_tags, theme)
|
||||
|
||||
await report_progress(
|
||||
"Gathering themes from configured nodes",
|
||||
[(node, _load(node, tags)) for node, tags in self._inputs.nodes],
|
||||
5,
|
||||
)
|
||||
|
||||
async def _load_urls_with_theme(self) -> None:
|
||||
for url, tags, theme in self._inputs.urls:
|
||||
if theme is not None:
|
||||
|
@ -143,3 +165,59 @@ async def _get_theme(url: str) -> Optional[str]:
|
|||
|
||||
_LOGGER.error(f"No theme found for url {url}")
|
||||
return None
|
||||
|
||||
|
||||
CAT_HOBO_JSON_SCRIPT = """
|
||||
for hobo_json in $(find /var/lib/combo/ -name hobo.json); do
|
||||
sudo cat $hobo_json
|
||||
echo ","
|
||||
done
|
||||
"""
|
||||
|
||||
|
||||
async def _get_node_urls(
|
||||
node: str,
|
||||
) -> AsyncIterable[UrlEntry]:
|
||||
process = await create_subprocess_shell(
|
||||
f"ssh {node} -C bash",
|
||||
stdin=PIPE,
|
||||
stdout=PIPE,
|
||||
stderr=PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate(CAT_HOBO_JSON_SCRIPT.encode("utf-8"))
|
||||
|
||||
if process.returncode != 0:
|
||||
print(stderr.decode("utf-8"))
|
||||
exit(1)
|
||||
|
||||
stdout_string = stdout.decode("utf-8").strip()
|
||||
config_array_content = f"[ {stdout_string[:-1]} ]"
|
||||
config_array = loads(config_array_content)
|
||||
|
||||
for tenant_config in config_array:
|
||||
services = tenant_config["services"]
|
||||
variables = tenant_config.get("variables", None)
|
||||
|
||||
if variables is None:
|
||||
continue
|
||||
|
||||
theme: Optional[str] = variables.get("theme", None)
|
||||
if theme is None:
|
||||
continue
|
||||
|
||||
for service in services:
|
||||
service_id = service.get("service-id", None)
|
||||
secondary = service.get("secondary", False)
|
||||
template_name = service.get("template_name", None)
|
||||
if (
|
||||
service_id not in ["combo", "wcs", "authentic"]
|
||||
or secondary
|
||||
or template_name == "portal-agent"
|
||||
):
|
||||
continue
|
||||
|
||||
base_url = service.get("base_url", None)
|
||||
if base_url:
|
||||
if service_id == "authentic":
|
||||
base_url = base_url + "login/"
|
||||
yield (base_url, [service_id], theme)
|
||||
|
|
Loading…
Reference in New Issue