feat: update community interfaces JSON builder with URL validation and fetching logic

scripts/build_community_interfaces_json.py

@@ -7,11 +7,11 @@ import argparse
 import json
 import sys
 import urllib.error
+import urllib.request
 from pathlib import Path
+from urllib.parse import urlparse
 
 from meshchatx.src.backend.community_interfaces_directory import (
-    DEFAULT_SUBMITTED_URL,
-    build_interfaces_from_directory_url,
     rows_from_payload,
     transform_directory_rows,
 )
@@ -19,6 +19,98 @@ from meshchatx.src.backend.community_interfaces_directory import (
 ROOT = Path(__file__).resolve().parents[1]
 OUT = ROOT / "meshchatx" / "src" / "backend" / "data" / "community_interfaces.json"
 
+DEFAULT_SUBMITTED_URL = (
+    "https://directory.rns.recipes/api/directory/submitted?status=online"
+)
+DEFAULT_DISCOVERED_URL = (
+    "https://directory.rns.recipes/api/directory/discovered?status=online"
+)
+DEFAULT_DIRECTORY_URLS = (DEFAULT_SUBMITTED_URL, DEFAULT_DISCOVERED_URL)
+_ALLOWED_HOST = "directory.rns.recipes"
+_MAX_FETCH_BYTES = 1 * 1024 * 1024
+_FETCH_HEADERS = {
+    "Accept": "application/json",
+    "User-Agent": "MeshChatX-community-interfaces-build/1.0 (+https://meshchatx.com/)",
+}
+
+
+def validate_directory_fetch_url(url: str) -> str:
+    if not url or not isinstance(url, str):
+        msg = "URL must be a non-empty string"
+        raise ValueError(msg)
+    parsed = urlparse(url.strip())
+    if parsed.scheme != "https":
+        msg = "Community directory URL must use https"
+        raise ValueError(msg)
+    netloc = parsed.netloc or ""
+    if "@" in netloc:
+        msg = "Community directory URL must not contain credentials"
+        raise ValueError(msg)
+    host = (parsed.hostname or "").lower()
+    if host != _ALLOWED_HOST:
+        msg = "Community directory URL host is not allowed"
+        raise ValueError(msg)
+    return url.strip()
+
+
+def fetch_directory_payload(url: str, *, timeout: float = 60.0) -> object:
+    resolved = validate_directory_fetch_url(url)
+    req = urllib.request.Request(resolved, headers=_FETCH_HEADERS, method="GET")
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        raw = resp.read(_MAX_FETCH_BYTES + 1)
+    if len(raw) > _MAX_FETCH_BYTES:
+        msg = f"Community directory download exceeds {_MAX_FETCH_BYTES} bytes"
+        raise ValueError(msg)
+    return json.loads(raw.decode("utf-8"))
+
+
+def _merge_directory_rows(row_lists: list[list]) -> list:
+    merged: list = []
+    seen: set[tuple] = set()
+    for rows in row_lists:
+        for row in rows:
+            if not isinstance(row, dict):
+                continue
+            key = (
+                str(row.get("name") or "").strip().lower(),
+                str(row.get("type") or "").strip().lower(),
+                str(row.get("host") or row.get("address") or "").strip().lower(),
+                str(row.get("port") or "").strip(),
+            )
+            if key in seen:
+                continue
+            seen.add(key)
+            merged.append(row)
+    return merged
+
+
+def build_interfaces_from_directory_url(
+    url: str | None = None,
+    *,
+    timeout: float = 60.0,
+) -> tuple[list[dict], str]:
+    if url is not None and str(url).strip():
+        resolved = validate_directory_fetch_url(url)
+        payload = fetch_directory_payload(resolved, timeout=timeout)
+        rows = rows_from_payload(payload)
+        return transform_directory_rows(rows), resolved
+
+    row_lists: list[list] = []
+    used: list[str] = []
+    errors: list[str] = []
+    for candidate in DEFAULT_DIRECTORY_URLS:
+        try:
+            payload = fetch_directory_payload(candidate, timeout=timeout)
+            row_lists.append(rows_from_payload(payload))
+            used.append(candidate)
+        except Exception as exc:
+            errors.append(f"{candidate}: {exc}")
+    if not used:
+        msg = "; ".join(errors) if errors else "No directory URLs configured"
+        raise ValueError(msg)
+    rows = _merge_directory_rows(row_lists)
+    return transform_directory_rows(rows), " + ".join(used)
+
 
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)