From ae22aa4c418771c904224f84dd1acdd153420de8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Corentin=20S=C3=A9chet?= <csechet@entrouvert.com>
Date: Tue, 12 Apr 2022 13:27:45 +0200
Subject: [PATCH] common: add ability to load urls from nodes

---
 frontools/config.py      |  6 ++-
 frontools/theme_index.py | 82 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/frontools/config.py b/frontools/config.py
index e6c4ec3..5e59973 100644
--- a/frontools/config.py
+++ b/frontools/config.py
@@ -9,7 +9,7 @@ from typing import Any, Iterable, Optional, Type, Union
 from xdg import xdg_config_dirs, xdg_config_home
 
 from frontools.sources import CachedSource, OverrideSource, Source
-from frontools.theme_index import ThemeIndex, UrlEntry
+from frontools.theme_index import ThemeIndex, UrlEntry, NodeEntry
 
 REMOTE_SOURCE_NAME = "remote"
 
@@ -91,6 +91,10 @@ class Config:
         """Add an url for a theme"""
         self._theme_index.add_urls(*urls)
 
+    def add_nodes(self, *nodes: NodeEntry) -> None:
+        """Add an url for a theme"""
+        self._theme_index.add_nodes(*nodes)
+
     def add_yaml(self, yaml_path: Union[str, Path]) -> None:
         """Load a yaml file containing dictionnary of urls to add as themes."""
         self._theme_index.add_yaml(Path(yaml_path))
diff --git a/frontools/theme_index.py b/frontools/theme_index.py
index d1cbc2d..5df26b8 100644
--- a/frontools/theme_index.py
+++ b/frontools/theme_index.py
@@ -1,9 +1,12 @@
 """Store themes and associated urls, providing ways to load them from several sources."""
+from asyncio import create_subprocess_shell
+from asyncio.subprocess import PIPE
+from json import loads
 from logging import getLogger
 from pathlib import Path
 from re import compile as re_compile
 from ssl import CERT_NONE, create_default_context
-from typing import Iterable, Optional
+from typing import AsyncIterable, Iterable, Optional
 
 from aiohttp import ClientSession
 from bs4 import BeautifulSoup
@@ -17,11 +20,13 @@ _LOGGER = getLogger(__file__)
 
 ThemeIndexData = dict[str, dict[str, list[str]]]
 UrlEntry = tuple[str, list[str], Optional[str]]  # (url, tags, theme name) tuples
+NodeEntry = tuple[str, list[str]]  # (server address, tags) tuples
 
 
 class _Inputs:
     urls: list[UrlEntry] = []
     yaml_files: list[Path] = []
+    nodes: list[NodeEntry] = []
 
 
 class ThemeIndex:
@@ -42,6 +47,10 @@ class ThemeIndex:
         """Add an url for a theme"""
         self._inputs.urls.extend(urls)
 
+    def add_nodes(self, *nodes: NodeEntry) -> None:
+        """Add an url for a theme"""
+        self._inputs.nodes.extend(nodes)
+
     def add_yaml(self, yaml_path: Path) -> None:
         """Load a yaml file containing dictionnary of urls to add as themes."""
         self._inputs.yaml_files.append(yaml_path)
@@ -52,6 +61,7 @@ class ThemeIndex:
             _Inputs.yaml_files.append(index_cache)
         else:
             await self._load_urls_without_theme()
+            await self._load_urls_from_nodes()
             with open(index_cache, "w") as index_cache_handle:
                 dump(self._themes, index_cache_handle)
 
@@ -71,11 +81,23 @@ class ThemeIndex:
             if theme is None
         ]
         await report_progress(
-            "Gathering themes from IMIO sites",
+            "Gathering themes from configured urls",
             tasks,
             10,
         )
 
+    async def _load_urls_from_nodes(self) -> None:
+        async def _load(node: str, node_tags: list[str]) -> None:
+            async for url, tags, theme in _get_node_urls(node):
+                assert theme is not None
+                self._register(url, tags + node_tags, theme)
+
+        await report_progress(
+            "Gathering themes from configured nodes",
+            [(node, _load(node, tags)) for node, tags in self._inputs.nodes],
+            5,
+        )
+
     async def _load_urls_with_theme(self) -> None:
         for url, tags, theme in self._inputs.urls:
             if theme is not None:
@@ -143,3 +165,59 @@ async def _get_theme(url: str) -> Optional[str]:
 
     _LOGGER.error(f"No theme found for url {url}")
     return None
+
+
+CAT_HOBO_JSON_SCRIPT = """
+for hobo_json in $(find /var/lib/combo/ -name hobo.json); do
+    sudo cat $hobo_json
+    echo ","
+done
+"""
+
+
+async def _get_node_urls(
+    node: str,
+) -> AsyncIterable[UrlEntry]:
+    process = await create_subprocess_shell(
+        f"ssh {node} -C bash",
+        stdin=PIPE,
+        stdout=PIPE,
+        stderr=PIPE,
+    )
+    stdout, stderr = await process.communicate(CAT_HOBO_JSON_SCRIPT.encode("utf-8"))
+
+    if process.returncode != 0:
+        print(stderr.decode("utf-8"))
+        exit(1)
+
+    stdout_string = stdout.decode("utf-8").strip()
+    config_array_content = f"[ {stdout_string[:-1]} ]"
+    config_array = loads(config_array_content)
+
+    for tenant_config in config_array:
+        services = tenant_config["services"]
+        variables = tenant_config.get("variables", None)
+
+        if variables is None:
+            continue
+
+        theme: Optional[str] = variables.get("theme", None)
+        if theme is None:
+            continue
+
+        for service in services:
+            service_id = service.get("service-id", None)
+            secondary = service.get("secondary", False)
+            template_name = service.get("template_name", None)
+            if (
+                service_id not in ["combo", "wcs", "authentic"]
+                or secondary
+                or template_name == "portal-agent"
+            ):
+                continue
+
+            base_url = service.get("base_url", None)
+            if base_url:
+                if service_id == "authentic":
+                    base_url = base_url + "login/"
+                yield (base_url, [service_id], theme)