Fix ctrld TOML generator and add pre-flight endpoint validation

TOML structural fix (critical):
- Remove wrapper [listener]/[network]/[upstream] headers; use flat
  dotted-key notation ([listener.0], [network.0], etc.) that ctrld's
  Go TOML v2 parser requires — the old nested style triggered a table
  redefinition panic at startup
- Add 'name' field to every [upstream.N] section (required by ctrld)
- Add [listener.0.policy] name field

DoH3 and protocol support:
- CtrldVlanProfile gains protocol (default "doh3") and endpoint_url
  fields; endpoint_url overrides the ControlD resolver_id URL if set
- Upstream type now uses the profile's protocol instead of hardcoded
  "doh" — enables DoH3 connection-pool reuse added in ctrld 2025

Endpoint pre-flight validation:
- New _validate_doh_endpoint(): sends RFC 8484 DoH GET query over
  plain HTTPS (works for DoH3 URLs too — ControlD serves both) and
  measures latency; no ctrld binary or Docker required
- New POST /api/ctrld/validate-endpoints: tests all profile endpoints,
  validates TOML syntax via tomllib (Python 3.11+), returns per-profile
  results + toml_preview
- ctrld_save_config now runs validation before writing anything and
  returns HTTP 400 with per-profile probe results on failure — configs
  are never pushed with a broken endpoint

OPNsense plugin verdict: documented in code — the os-controld plugin
kills Unbound and breaks OPNsense DNS advertisement; SSH-based deploy
with our own TOML remains the correct path

https://claude.ai/code/session_01JR2EMK7rwrZJowpstcaxQ6

switch_backend.py

@@ -2001,28 +2001,33 @@ def _ctrld_config_path() -> _Path:
 def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
                       local_resolver: str = "") -> str:
     """
-    Build a ctrld.toml in the correct format — table notation, not TOML arrays.
+    Build a ctrld.toml using flat dotted-key section headers — the only format
+    ctrld's Go TOML parser accepts without a redefinition panic.
 
-    The correct ctrld format uses [listener.0], [network.N], [upstream.N] table
+    WRONG (triggers redefinition panic in Go TOML v2):
-    sections, NOT [[listener]] / [[upstream]] / [[rule]] array tables.  Source
+        [listener]
-    VLAN routing is done via [network.N] sections (CIDR-based) referenced in the
+          [listener.0]   <- redefines the already-open table
-    [listener.0.policy].networks array.  Domain-specific overrides go in .rules.
+    CORRECT:
+        [listener.0]     <- opens the subtable directly, no parent wrapper
 
-    vlan_profiles: list of { vlan_id, name, subnet, resolver_id }
+    vlan_profiles: list of {
-    local_domain:  suffix for internal hostnames (default 'lan')
+        vlan_id, name, subnet,
-    local_resolver: if set (e.g. '127.0.0.1:5353'), adds split-horizon upstream
+        resolver_id,      # ControlD Resolver ID (path suffix) used when
-                    and rules so *.lan / *.local go to the local resolver instead
+                           # endpoint_url is empty
-                    of Control D — keeps .lan names working for all VLAN clients.
+        endpoint_url,     # full DoH/DoH3 URL — overrides resolver_id if set
-
+        protocol,         # "doh3" (default) | "doh" | "dot" | "doq" | "legacy"
-    Control D bootstrap IP 76.76.2.0 is used for cold-start before DoH is up.
+    }
+    local_resolver: if set (e.g. "127.0.0.1:5353"), adds split-horizon rules
+                    so *.lan / *.local go to the local resolver.
     """
-    BOOTSTRAP = "76.76.2.0"   # Control D anycast — required for cold-start
+    BOOTSTRAP = "76.76.2.0"   # ControlD anycast — needed for cold-start
 
-    active = [vp for vp in vlan_profiles if vp.get("resolver_id", "").strip()]
+    active = [vp for vp in vlan_profiles
+              if vp.get("resolver_id", "").strip() or vp.get("endpoint_url", "").strip()]
 
     lines = [
         "# ctrld configuration — generated by Avaya 59100GTS-PWR+ Switch Manager",
-        "# Documentation: https://docs.controld.com/docs/ctrld",
+        "# Docs: https://docs.controld.com/docs/ctrld",
         "",
         "[service]",
         "  log_level = 'info'",
@@ -2030,89 +2035,95 @@ def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
         "",
     ]
 
-    # ── Listener with per-VLAN policy ────────────────────────────────────────
+    # ── Listener — flat header, NO parent [listener] wrapper ─────────────────
-    # The policy.networks array maps each [network.N] to one [upstream.N].
-    # This is how ctrld routes different VLAN subnets to different profiles.
     lines += [
-        "[listener]",
+        "[listener.0]",
-        "  [listener.0]",
+        "  ip = '0.0.0.0'",
-        "    ip = '0.0.0.0'",
+        "  port = 53",
-        "    port = 53",
+        "",
-        "    [listener.0.policy]",
+        "  [listener.0.policy]",
+        "    name = 'VLAN Policy'",
     ]
 
     if active:
-        net_entries = [f"        " + "{ " + f"'network.{i}' = ['upstream.{i}']" + " },"
+        net_entries = [
-                       for i in range(len(active))]
+            "      { " + f"'network.{i}' = ['upstream.{i}']" + " },"
-        lines += ["      networks = ["] + net_entries + ["      ]"]
+            for i in range(len(active))
+        ]
+        lines += ["    networks = ["] + net_entries + ["    ]"]
     else:
-        lines += ["      networks = []"]
+        lines += ["    networks = []"]
 
-    # Domain-specific rules (split-horizon for .lan / .local → local resolver)
     if local_resolver:
         domain_suffix = local_domain.strip(".")
         lines += [
-            "      rules = [",
+            "    rules = [",
-            f"        " + "{ " + f"'*.{domain_suffix}' = ['upstream.local']" + " },",
+            "      { " + f"'*.{domain_suffix}' = ['upstream.local']" + " },",
-            "        " + "{ " + "'*.local' = ['upstream.local']" + " },",
+            "      { " + "'*.local' = ['upstream.local']" + " },",
-            "      ]",
+            "    ]",
         ]
     else:
-        lines += ["      rules = []"]
+        lines += ["    rules = []"]
 
     lines += [""]
 
-    # ── Network sections — one per VLAN ──────────────────────────────────────
+    # ── Network sections — flat headers, one per active VLAN ─────────────────
-    if active:
-        lines += ["[network]"]
-        for i, vp in enumerate(active):
-            vid    = vp["vlan_id"]
-            name   = vp.get("name", f"VLAN{vid}")
-            subnet = vp.get("subnet", f"192.168.{vid}.0/24")
-            lines += [
-                f"  # VLAN {vid} — {name}",
-                f"  [network.{i}]",
-                f"    name = '{name}'",
-                f"    cidrs = ['{subnet}']",
-                "",
-            ]
-
-    # ── Upstream sections — one per VLAN plus optional local ──────────────────
-    lines += ["[upstream]"]
     for i, vp in enumerate(active):
-        vid = vp["vlan_id"]
+        vid    = vp["vlan_id"]
-        rid = vp["resolver_id"].strip()
+        name   = vp.get("name", f"VLAN{vid}")
+        subnet = vp.get("subnet", f"192.168.{vid}.0/24")
         lines += [
-            f"  # VLAN {vid} — {vp.get('name', '')}",
+            f"# VLAN {vid} — {name}",
-            f"  [upstream.{i}]",
+            f"[network.{i}]",
-            f"    type = 'doh'",
+            f"  name = '{name}'",
-            f"    endpoint = 'https://dns.controld.com/{rid}'",
+            f"  cidrs = ['{subnet}']",
-            f"    bootstrap_ip = '{BOOTSTRAP}'",
-            f"    timeout = 5000",
             "",
         ]
 
-    # Optional local resolver for split-horizon .lan resolution (dnsmasq/Unbound)
+    # ── Upstream sections — flat headers, one per active VLAN ────────────────
+    for i, vp in enumerate(active):
+        vid      = vp["vlan_id"]
+        name     = vp.get("name", f"VLAN{vid}")
+        protocol = (vp.get("protocol") or "doh3").strip()
+        endpoint = (
+            vp.get("endpoint_url", "").strip()
+            or f"https://dns.controld.com/{vp['resolver_id'].strip()}"
+        )
+        lines += [
+            f"# VLAN {vid} — {name}",
+            f"[upstream.{i}]",
+            f"  name = 'VLAN {vid} {name}'",
+            f"  type = '{protocol}'",
+            f"  endpoint = '{endpoint}'",
+            f"  bootstrap_ip = '{BOOTSTRAP}'",
+            f"  timeout = 5000",
+            "",
+        ]
+
+    # Optional local split-horizon resolver
     if local_resolver:
         lines += [
-            f"  # Local resolver — handles *.{local_domain} and *.local",
+            f"# Local resolver — *.{local_domain} and *.local",
-            f"  # dnsmasq on port 5353 (Docker) or Unbound on 127.0.0.1:5353 (OPNsense)",
+            "[upstream.local]",
-            f"  [upstream.local]",
+            "  name = 'Local Resolver'",
-            f"    type = 'legacy'",
+            "  type = 'legacy'",
-            f"    endpoint = '{local_resolver}'",
+            f"  endpoint = '{local_resolver}'",
-            f"    timeout = 2000",
+            "  timeout = 2000",
             "",
         ]
 
     return "\n".join(lines)
 
-# ── ctrld API models ────────────────────────────────────────────────────────
+
+# ── ctrld API models ─────────────────────────────────────────────────────────
 
 class CtrldVlanProfile(BaseModel):
-    vlan_id:     int
+    vlan_id:      int
-    name:        str
+    name:         str
-    subnet:      str
+    subnet:       str
-    resolver_id: str
+    resolver_id:  str                      # ControlD Resolver ID (path suffix)
+    endpoint_url: Optional[str] = ""       # full URL — overrides resolver_id if set
+    protocol:     Optional[str] = "doh3"  # doh3 | doh | dot | doq | legacy
 
 class CtrldConfig(BaseModel):
     mode:          str   # "local" | "opnsense" | "manual"
@@ -2127,7 +2138,7 @@ class CtrldUpdateProfile(BaseModel):
     token: str
     vlan_profiles: list[CtrldVlanProfile]
 
-# ── ctrld endpoints ─────────────────────────────────────────────────────────
+# ── ctrld endpoints ───────────────────────────────────────────────────────────
 
 @app.get("/api/ctrld/status")
 def ctrld_status():
@@ -2144,6 +2155,119 @@ def ctrld_status():
         "docs_url":       "https://docs.controld.com/docs/ctrld",
     }
 
+def _validate_doh_endpoint(endpoint_url: str) -> dict:
+    """
+    Test a DoH endpoint using a plain HTTPS GET query (RFC 8484).
+
+    Works for both DoH and DoH3 endpoints — ControlD and most public resolvers
+    serve DoH over HTTPS/2 at the same URL they use for DoH3, so a successful
+    HTTP response proves the URL is live and well-formed before ctrld ever
+    touches it.
+
+    Sends:  GET {url}?dns=<base64url(A? ping.controld.com)>
+    Expects: 200, Content-Type: application/dns-message
+
+    Returns: {ok, latency_ms, status_code, error}
+    """
+    import struct, base64, time, urllib.request, urllib.error
+
+    # Build a minimal DNS A query for "ping.controld.com" in wire format
+    def _make_query(domain: str = "ping.controld.com") -> bytes:
+        hdr = struct.pack(">HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0)
+        qname = b""
+        for label in domain.rstrip(".").split("."):
+            qname += bytes([len(label)]) + label.encode()
+        qname += b"\x00"
+        qtype = struct.pack(">HH", 1, 1)   # A IN
+        return hdr + qname + qtype
+
+    dns_bytes    = _make_query()
+    dns_b64      = base64.urlsafe_b64encode(dns_bytes).rstrip(b"=").decode()
+    test_url     = f"{endpoint_url.rstrip('/')}?dns={dns_b64}"
+    t0           = time.monotonic()
+
+    try:
+        req = urllib.request.Request(
+            test_url,
+            headers={"Accept": "application/dns-message"},
+        )
+        with urllib.request.urlopen(req, timeout=8) as resp:
+            latency_ms = int((time.monotonic() - t0) * 1000)
+            ct         = resp.headers.get("Content-Type", "")
+            body       = resp.read(12)   # just enough to check it's a DNS response
+            ok         = resp.status == 200 and "dns-message" in ct
+            return {
+                "ok":         ok,
+                "latency_ms": latency_ms,
+                "status_code": resp.status,
+                "error":      "" if ok else f"Unexpected Content-Type: {ct}",
+            }
+    except urllib.error.HTTPError as e:
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        return {"ok": False, "latency_ms": latency_ms,
+                "status_code": e.code, "error": str(e)}
+    except Exception as e:
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        return {"ok": False, "latency_ms": latency_ms,
+                "status_code": 0, "error": str(e)}
+
+
+class CtrldValidateRequest(BaseModel):
+    vlan_profiles: list[CtrldVlanProfile]
+
+
+@app.post("/api/ctrld/validate-endpoints")
+def ctrld_validate_endpoints(body: CtrldValidateRequest):
+    """
+    Test each profile's DoH/DoH3 endpoint URL before writing any config.
+    Returns per-profile results plus an overall ok flag.
+    Intended to be called from the UI before calling save-config.
+    """
+    results = []
+    for vp in body.vlan_profiles:
+        p      = vp.dict()
+        url    = (p.get("endpoint_url") or "").strip()
+        rid    = (p.get("resolver_id") or "").strip()
+        if not url and rid:
+            url = f"https://dns.controld.com/{rid}"
+        if not url:
+            results.append({
+                "vlan_id": p["vlan_id"],
+                "name":    p.get("name", ""),
+                "ok":      False,
+                "error":   "No endpoint URL or resolver_id provided",
+            })
+            continue
+        probe = _validate_doh_endpoint(url)
+        results.append({
+            "vlan_id":    p["vlan_id"],
+            "name":       p.get("name", ""),
+            "url":        url,
+            **probe,
+        })
+
+    # Also validate the generated TOML can be parsed (catches structural issues)
+    toml_ok    = True
+    toml_error = ""
+    try:
+        import sys
+        if sys.version_info >= (3, 11):
+            import tomllib
+            tomllib.loads(_build_ctrld_toml([p.dict() for p in body.vlan_profiles]))
+        # tomllib not available in 3.10 — skip structural check
+    except Exception as e:
+        toml_ok    = False
+        toml_error = str(e)
+
+    return {
+        "all_ok":      all(r["ok"] for r in results) and toml_ok,
+        "results":     results,
+        "toml_ok":     toml_ok,
+        "toml_error":  toml_error,
+        "toml_preview": _build_ctrld_toml([p.dict() for p in body.vlan_profiles]),
+    }
+
+
 @app.get("/api/ctrld/toml-preview")
 def ctrld_toml_preview():
     """Generate and return the ctrld.toml without installing it."""
@@ -2158,21 +2282,35 @@ def ctrld_toml_preview():
 def ctrld_save_config(body: CtrldInstallRequest):
     """
     Save ctrld configuration.
+    Validates all endpoints before writing anything — returns 400 with per-profile
+    probe results if any endpoint is unreachable so the user can fix it first.
     For 'local' mode: installs ctrld on this machine, writes config, starts service.
     For 'opnsense' mode: generates the SSH install command.
     For 'manual' mode: saves config for reference, generates toml only.
     """
     require_session(body.token)
 
+    profiles = [p.dict() for p in body.config.vlan_profiles]
+
+    # ── Endpoint validation — block on failure ────────────────────────────────
+    validation = ctrld_validate_endpoints(
+        CtrldValidateRequest(vlan_profiles=body.config.vlan_profiles)
+    )
+    if not validation["all_ok"]:
+        raise HTTPException(400, {
+            "message":  "One or more DNS endpoints failed validation — fix before saving",
+            "results":  validation["results"],
+            "toml_error": validation.get("toml_error", ""),
+        })
+
     cfg_dict = {
         "mode":          body.config.mode,
-        "vlan_profiles": [p.dict() for p in body.config.vlan_profiles],
+        "vlan_profiles": profiles,
         "opnsense_host": body.config.opnsense_host,
     }
     _save_ctrld_cfg(cfg_dict)
 
-    profiles = [p.dict() for p in body.config.vlan_profiles]
+    toml = _build_ctrld_toml(profiles)
-    toml     = _build_ctrld_toml(profiles)
 
     if body.config.mode == "local":
         return _ctrld_install_local(toml, profiles)