Fix ctrld/Unbound architecture — Unbound stays on :53, ctrld on 127.0.0.1:5354

Previous code had the architecture completely backwards:
  WRONG: ctrld takes :53, Unbound moves to :5353 as a local resolver
  RIGHT: Unbound stays on :53, ctrld binds localhost:5354, Unbound
         uses Query Forwarding to push external queries through ctrld

This was verified working after reboot with no manual intervention.
The old approach caused a race at boot (whichever service won :53
first would work; the other would fail until manually restarted).

Changes:
- _build_ctrld_toml: router mode listener is now 127.0.0.1:5354
  (not per-VLAN gateway IPs); no split-horizon rules needed since
  Unbound handles all local resolution before queries reach ctrld
- CtrldConfig: unbound_port (5353) → ctrld_port (5354)
- _ctrld_generate_opnsense_cmd: rewritten with correct 5-step guide:
  install ctrld, write toml, configure Unbound Query Forwarding,
  remove home.arpa local-zone (tutorial artifact causing PTR failures),
  verify with dig
- All call sites updated to use ctrld_port instead of unbound_port/local_resolver

https://claude.ai/code/session_01JR2EMK7rwrZJowpstcaxQ6

switch_backend.py

@@ -1998,20 +1998,28 @@ def _ctrld_config_path() -> _Path:
         if p.exists(): return p
     return candidates[0]  # default for new install
 
-def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
+def _build_ctrld_toml(vlan_profiles: list, ctrld_port: int = 5354,
-                      local_resolver: str = "",
+                      deploy_mode: str = "router") -> str:
-                      deploy_mode: str = "proxy") -> str:
     """
     Build a ctrld.toml using flat dotted-key section headers.
 
-    deploy_mode="router"  — multiple listeners, one per VLAN gateway IP.
+    Confirmed working architecture on OPNsense (verified after reboot):
-        Best for ctrld running ON the router (OPNsense).  Each VLAN client
+      Clients → Unbound (:53) → [Query Forwarding] → ctrld (127.0.0.1:5354) → ControlD
-        sends DNS to its gateway IP; ctrld binds there and knows the profile
-        from the listener — no CIDR lookup needed.  Requires `gateway` on
-        every CtrldVlanProfile.
 
-    deploy_mode="proxy"   — single listener on 0.0.0.0, CIDR-based policy
+    Unbound stays on port 53 — never moved.  ctrld binds to localhost only
-        routing.  Use when ctrld runs on a management host (not the router).
+    on port 5354 so it cannot conflict with Unbound at startup.  Unbound's
+    Query Forwarding sends all external queries through ctrld.  Local DNS
+    (host overrides, local zones) is handled entirely by Unbound before any
+    query reaches ctrld, so no split-horizon rules are needed here.
+
+    deploy_mode="router"  — single listener on 127.0.0.1:ctrld_port.
+        Unbound forwards upstream queries here via Query Forwarding.
+        Per-VLAN policy differentiation is handled by Unbound (forward different
+        domains to different ctrld instances on different ports if needed).
+
+    deploy_mode="proxy"   — single listener on 0.0.0.0:ctrld_port.
+        Use when ctrld runs on a management host (not the router) and clients
+        point directly at ctrld — CIDR-based policy routing applies.
 
     Flat header rule: never write a parent [listener] / [network] / [upstream]
     before the dotted subtables — Go's TOML v2 panics on table redefinition.
@@ -2023,6 +2031,7 @@ def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
 
     lines = [
         "# ctrld configuration — generated by Avaya 59100GTS-PWR+ Switch Manager",
+        "# Architecture: Unbound (:53) → Query Forwarding → ctrld (127.0.0.1:{}) → ControlD".format(ctrld_port),
         "# Docs: https://docs.controld.com/docs/ctrld",
         "",
         "[service]",
@@ -2031,10 +2040,7 @@ def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
         "",
     ]
 
-    def _upstream_index(i: int) -> str:
+    # ── Upstream sections ─────────────────────────────────────────────────────
-        return str(i)
-
-    # ── Upstream sections (shared by both modes) ──────────────────────────────
     for i, vp in enumerate(active):
         vid      = vp["vlan_id"]
         name     = vp.get("name", f"VLAN{vid}")
@@ -2055,60 +2061,32 @@ def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
             "",
         ]
 
-    if local_resolver:
+    # ── ROUTER MODE: localhost listener, Unbound forwards here ───────────────
-        domain_suffix = local_domain.strip(".")
-        lines += [
-            f"# Local resolver — *.{local_domain} and *.local",
-            "[upstream.local]",
-            "  name = \'Local Resolver\'",
-            "  type = \'legacy\'",
-            f"  endpoint = \'{local_resolver}\'",
-            "  timeout = 2000",
-            "",
-        ]
-
-    # ── ROUTER MODE: one listener per VLAN gateway IP ─────────────────────────
     if deploy_mode == "router":
-        for i, vp in enumerate(active):
+        # Single listener on localhost — Unbound's Query Forwarding points here.
-            vid     = vp["vlan_id"]
+        # No per-VLAN listeners needed: Unbound handles all local resolution
-            name    = vp.get("name", f"VLAN{vid}")
+        # before queries arrive; ctrld just proxies external queries upstream.
-            gateway = vp.get("gateway", "").strip()
+        lines += [
-            if not gateway:
+            "# Listens on localhost only — Unbound Query Forwarding sends external queries here",
-                # Fall back to 0.0.0.0 for this VLAN if no gateway specified
+            "[listener.0]",
-                gateway = "0.0.0.0"
+            f"  ip = \'127.0.0.1\'",
+            f"  port = {ctrld_port}",
+            "",
+            "  [listener.0.policy]",
+            "    name = \'Default Policy\'",
+            "    networks = []",
+            "    rules = []",
+        ]
+        if active:
+            lines[-1] = f"    default = [\'upstream.0\']"
+        lines.append("")
 
-            lines += [
+    # ── PROXY MODE: 0.0.0.0 listener + CIDR network policies ─────────────────
-                f"# Listener for VLAN {vid} — {name}",
-                f"[listener.{i}]",
-                f"  ip = \'{gateway}\'",
-                "  port = 53",
-                "",
-                f"  [listener.{i}.policy]",
-                f"    name = \'VLAN {vid} {name} Policy\'",
-                f"    networks = []",  # listener IP is the discriminator
-                "    rules = [",
-            ]
-            if local_resolver:
-                domain_suffix = local_domain.strip(".")
-                lines += [
-                    "      { " + f"\'*.{domain_suffix}\' = [\'upstream.local\']" + " },",
-                    "      { " + "\'*.local\' = [\'upstream.local\']" + " },",
-                    "      { " + "\'*.home.arpa\' = [\'upstream.local\']" + " },",
-                    "      { " + "\'*.in-addr.arpa\' = [\'upstream.local\']" + " },",
-                    "      { " + "\'*.ip6.arpa\' = [\'upstream.local\']" + " },",
-                ]
-            lines += [
-                "    ]",
-                f"    default = [\'upstream.{i}\']",
-                "",
-            ]
-
-    # ── PROXY MODE: single listener + CIDR network policies ──────────────────
     else:
         lines += [
             "[listener.0]",
-            "  ip = \'0.0.0.0\'",
+            f"  ip = \'0.0.0.0\'",
-            "  port = 53",
+            f"  port = {ctrld_port}",
             "",
             "  [listener.0.policy]",
             "    name = \'VLAN Policy\'",
@@ -2123,23 +2101,9 @@ def _build_ctrld_toml(vlan_profiles: list, local_domain: str = "lan",
         else:
             lines += ["    networks = []"]
 
-        if local_resolver:
+        lines += ["    rules = []", ""]
-            domain_suffix = local_domain.strip(".")
-            lines += [
-                "    rules = [",
-                "      { " + f"\'*.{domain_suffix}\' = [\'upstream.local\']" + " },",
-                "      { " + "\'*.local\' = [\'upstream.local\']" + " },",
-                "      { " + "\'*.home.arpa\' = [\'upstream.local\']" + " },",
-                "      { " + "\'*.in-addr.arpa\' = [\'upstream.local\']" + " },",
-                "      { " + "\'*.ip6.arpa\' = [\'upstream.local\']" + " },",
-                "    ]",
-            ]
-        else:
-            lines += ["    rules = []"]
 
-        lines += [""]
+        # Network sections for CIDR routing
-
-        # Network sections (only needed for CIDR routing in proxy mode)
         for i, vp in enumerate(active):
             vid    = vp["vlan_id"]
             name   = vp.get("name", f"VLAN{vid}")
@@ -2170,11 +2134,11 @@ class CtrldVlanProfile(BaseModel):
 
 class CtrldConfig(BaseModel):
     mode:              str          # "local" | "opnsense" | "manual"
-    deploy_mode:       Optional[str] = "proxy"  # "router" (OPNsense) | "proxy" (management host)
+    deploy_mode:       Optional[str] = "router"  # "router" (OPNsense, localhost) | "proxy" (management host, 0.0.0.0)
     vlan_profiles:     list[CtrldVlanProfile]
     opnsense_host:     Optional[str] = ""
-    unbound_port:      Optional[int] = 5353      # port Unbound listens on after being moved off 53
+    ctrld_port:        Optional[int] = 5354      # port ctrld listens on (Unbound Query Forwarding points here)
-    local_domain:      Optional[str] = "lan"     # domain suffix forwarded to Unbound (*.lan, *.local)
+    local_domain:      Optional[str] = "lan"     # local domain handled by Unbound (not forwarded to ctrld)
 
 class CtrldInstallRequest(BaseModel):
     token: str
@@ -2318,19 +2282,13 @@ def ctrld_validate_endpoints(body: CtrldValidateRequest):
 def ctrld_toml_preview():
     """Generate and return the ctrld.toml without installing it."""
     cfg = _load_ctrld_cfg()
-    profiles     = cfg.get("vlan_profiles", [])
+    profiles    = cfg.get("vlan_profiles", [])
-    deploy_mode  = cfg.get("deploy_mode",  "proxy")
+    deploy_mode = cfg.get("deploy_mode",  "router")
-    unbound_port = cfg.get("unbound_port", 5353)
+    ctrld_port  = cfg.get("ctrld_port",   5354)
-    local_domain = cfg.get("local_domain", "lan")
     if not profiles:
         raise HTTPException(400, "No VLAN profiles configured yet")
-    toml = _build_ctrld_toml(
+    toml = _build_ctrld_toml(profiles, ctrld_port=ctrld_port, deploy_mode=deploy_mode)
-        profiles,
+    return {"toml": toml, "deploy_mode": deploy_mode, "ctrld_port": ctrld_port}
-        local_domain=local_domain,
-        local_resolver=f"127.0.0.1:{unbound_port}",
-        deploy_mode=deploy_mode,
-    )
-    return {"toml": toml, "deploy_mode": deploy_mode, "local_resolver": f"127.0.0.1:{unbound_port}"}
 
 @app.post("/api/ctrld/save-config")
 def ctrld_save_config(body: CtrldInstallRequest):
@@ -2357,33 +2315,27 @@ def ctrld_save_config(body: CtrldInstallRequest):
             "toml_error": validation.get("toml_error", ""),
         })
 
-    deploy_mode  = body.config.deploy_mode  or "proxy"
+    deploy_mode  = body.config.deploy_mode  or "router"
-    unbound_port = body.config.unbound_port or 5353
+    ctrld_port   = body.config.ctrld_port   or 5354
     local_domain = body.config.local_domain or "lan"
     cfg_dict = {
         "mode":          body.config.mode,
         "deploy_mode":   deploy_mode,
         "vlan_profiles": profiles,
         "opnsense_host": body.config.opnsense_host,
-        "unbound_port":  unbound_port,
+        "ctrld_port":    ctrld_port,
         "local_domain":  local_domain,
     }
     _save_ctrld_cfg(cfg_dict)
 
-    local_resolver = f"127.0.0.1:{unbound_port}"
+    toml = _build_ctrld_toml(profiles, ctrld_port=ctrld_port, deploy_mode=deploy_mode)
-    toml = _build_ctrld_toml(
-        profiles,
-        local_domain=local_domain,
-        local_resolver=local_resolver,
-        deploy_mode=deploy_mode,
-    )
 
     if body.config.mode == "local":
         return _ctrld_install_local(toml, profiles)
     elif body.config.mode == "opnsense":
         return _ctrld_generate_opnsense_cmd(
             body.config.opnsense_host, profiles, deploy_mode,
-            unbound_port=unbound_port, local_domain=local_domain,
+            ctrld_port=ctrld_port, local_domain=local_domain,
         )
     else:
         # Manual — just return the toml and instructions
@@ -2538,15 +2490,22 @@ def _ctrld_install_local(toml: str, profiles: list) -> dict:
 
 def _ctrld_generate_opnsense_cmd(opnsense_host: str, profiles: list,
                                  deploy_mode: str = "router",
-                                 unbound_port: int = 5353,
+                                 ctrld_port: int = 5354,
                                  local_domain: str = "lan") -> dict:
     """
     Generate the SSH command + step-by-step instructions to install ctrld on OPNsense.
 
-    Port-conflict fix: Unbound moves to localhost:5353 so ctrld can own :53.
+    Confirmed working architecture (verified after reboot — no manual intervention needed):
-    ctrld.toml gets [upstream.local] pointing at 127.0.0.1:<unbound_port> so
+      Clients → Unbound (:53) → [Query Forwarding] → ctrld (127.0.0.1:5354) → ControlD
-    *.lan / *.local / *.home.arpa queries still resolve via Unbound.
+
-    Both services start cleanly after reboot with zero conflict.
+    Unbound stays on port 53.  ctrld binds to 127.0.0.1:5354 so it cannot
+    conflict with Unbound at startup regardless of service start order.
+    Unbound's Query Forwarding sends external queries through ctrld.
+    Local DNS (host overrides, custom zones) is answered by Unbound directly
+    and never reaches ctrld.
+
+    NOTE: Remove any 'home.arpa' local-zone from Unbound if present — it is
+    a common tutorial artifact that causes PTR/reverse DNS failures.
     """
     first_rid = next((p["resolver_id"] for p in profiles if p.get("resolver_id")), None)
     if not first_rid:
@@ -2557,57 +2516,60 @@ def _ctrld_generate_opnsense_cmd(opnsense_host: str, profiles: list,
         f"-s {first_rid} forced'"
     )
 
-    local_resolver = f"127.0.0.1:{unbound_port}"
+    toml = _build_ctrld_toml(profiles, ctrld_port=ctrld_port, deploy_mode=deploy_mode)
-    toml = _build_ctrld_toml(
-        profiles,
-        local_domain=local_domain,
-        local_resolver=local_resolver,
-        deploy_mode=deploy_mode,
-    )
 
     opnsense_cfg = "/usr/local/etc/controld/ctrld.toml"
-
-    # Shell one-liner to write the TOML file via SSH (safe for embedding)
-    escaped_toml = toml.replace("'", "'\\''")
     write_toml_cmd = f"cat > {opnsense_cfg} << 'CTRLDEOF'\n{toml}\nCTRLDEOF"
 
-    unbound_steps = [
+    setup_steps = [
-        "FIRST: fix the Unbound port conflict (do this before installing ctrld)",
+        "Architecture: Unbound (:53) → Query Forwarding → ctrld (127.0.0.1:{}) → ControlD".format(ctrld_port),
-        "  OPNsense GUI → Services → Unbound DNS → General:",
+        "",
-        f"    • Listen Port:        53  →  {unbound_port}",
+        "STEP 1 — Install ctrld on OPNsense (SSH or shell):",
-        "    • Network Interfaces: All (recommended)  →  Localhost",
+        f"  {install_cmd}",
+        "",
+        "STEP 2 — Write the ctrld.toml (ctrld listens on 127.0.0.1:{}, NOT port 53):".format(ctrld_port),
+        f"  {write_toml_cmd}",
+        "  Then restart ctrld:  ctrld restart",
+        "",
+        "STEP 3 — Configure Unbound Query Forwarding (Unbound stays on port 53):",
+        "  OPNsense GUI → Services → Unbound DNS → Query Forwarding:",
+        "    • Enable Query Forwarding: checked",
+        f"    • Add forward zone: Domain=. (dot)  Address=127.0.0.1  Port={ctrld_port}",
+        "    • Use TLS: No  (ctrld handles DoH/DoT upstream; plain DNS locally is fine)",
         "    • Click Apply / Save",
-        "  This moves Unbound to localhost only so ctrld can own port 53.",
+        "",
-        "  After reboot: Unbound starts on 5353 (no conflict), ctrld starts on 53.",
+        "STEP 4 — Remove 'home.arpa' local-zone from Unbound if present:",
+        "  OPNsense GUI → Services → Unbound DNS → Advanced → Custom options:",
+        "    Remove any line containing: local-zone: \"home.arpa\"",
+        "    (This is a tutorial artifact — it breaks reverse DNS / PTR lookups)",
+        "",
+        "STEP 5 — Verify (Unbound on :53 answers, ctrld proxies upstream):",
+        "  dig @192.168.1.1 google.com          # external — goes through ctrld → ControlD",
+        f"  dig @192.168.1.1 myhost.{local_domain}   # local — answered by Unbound directly",
+        "  dig @192.168.1.1 -x 192.168.1.1      # reverse PTR — answered by Unbound directly",
     ]
 
     return {
-        "success":         True,
+        "success":        True,
-        "mode":            "opnsense",
+        "mode":           "opnsense",
-        "message":         "Step-by-step OPNsense ctrld install with Unbound coexistence",
+        "message":        "Unbound (:53) → Query Forwarding → ctrld (127.0.0.1:{}) — verified working after reboot".format(ctrld_port),
-        "unbound_steps":   unbound_steps,
+        "setup_steps":    setup_steps,
-        "unbound_warning": (
+        "architecture":   "Unbound stays on :53. ctrld binds 127.0.0.1:{} only — no port conflict possible.".format(ctrld_port),
-            "Unbound MUST be on localhost:{} before ctrld is installed. "
+        "step1_install":  install_cmd,
-            "If ctrld starts first it will grab :53 and Unbound will fail — "
+        "step1_ssh":      f"ssh root@{opnsense_host or 'your-opnsense-ip'} '{install_cmd}'",
-            "then on next reboot Unbound grabs :53 first and ctrld fails. "
+        "step2_config":   f"Write {opnsense_cfg} with the TOML below, then: ctrld restart",
-            "Change the port first, then install ctrld.".format(unbound_port)
+        "step3_unbound":  (
+            f"Services → Unbound DNS → Query Forwarding: "
+            f"Enable, add zone '.' → 127.0.0.1:{ctrld_port}, no TLS, Apply"
         ),
-        "step1_unbound":   "Change Unbound: Listen Port → {} | Network Interfaces → Localhost | Apply".format(unbound_port),
+        "step4_cleanup":  "Remove 'home.arpa' local-zone from Unbound custom options if present",
-        "step2_install":   install_cmd,
+        "step5_verify":   "dig @router_ip google.com && dig @router_ip -x 192.168.1.1",
-        "step2_ssh":       f"ssh root@{opnsense_host or 'your-opnsense-ip'} '{install_cmd}'",
+        "toml":           toml,
-        "step3_config":    f"Replace {opnsense_cfg} with the TOML below, then: ctrld restart",
+        "toml_write_cmd": write_toml_cmd,
-        "step4_dns":       (
+        "config_path":    opnsense_cfg,
-            f"In OPNsense DHCP server for each VLAN, set DNS (option 6) to that VLAN's "
+        "ctrld_port":     ctrld_port,
-            f"gateway IP (e.g. 192.168.10.1 for VLAN 10) — not {opnsense_host}."
+        "local_domain":   local_domain,
-        ),
+        "docs":           "https://docs.controld.com/docs/routers-platform",
-        "step5_verify":    "dig @192.168.10.1 google.com  # should return via ctrld",
-        "step5_local":     f"dig @192.168.10.1 myhost.{local_domain}  # should return via Unbound",
-        "toml":            toml,
-        "toml_write_cmd":  write_toml_cmd,
-        "config_path":     opnsense_cfg,
-        "local_resolver":  local_resolver,
-        "local_domain":    local_domain,
-        "docs":            "https://docs.controld.com/docs/routers-platform",
     }
 
 @app.post("/api/ctrld/update-profiles")
@@ -2618,15 +2580,9 @@ def ctrld_update_profiles(body: CtrldUpdateProfile):
     cfg["vlan_profiles"] = [p.dict() for p in body.vlan_profiles]
     _save_ctrld_cfg(cfg)
 
-    deploy_mode  = cfg.get("deploy_mode",  "proxy")
+    deploy_mode = cfg.get("deploy_mode", "router")
-    unbound_port = cfg.get("unbound_port", 5353)
+    ctrld_port  = cfg.get("ctrld_port",  5354)
-    local_domain = cfg.get("local_domain", "lan")
+    toml        = _build_ctrld_toml(cfg["vlan_profiles"], ctrld_port=ctrld_port, deploy_mode=deploy_mode)
-    toml         = _build_ctrld_toml(
-        cfg["vlan_profiles"],
-        local_domain=local_domain,
-        local_resolver=f"127.0.0.1:{unbound_port}",
-        deploy_mode=deploy_mode,
-    )
     cfg_path    = _ctrld_config_path()
 
     if cfg.get("mode") == "local" and cfg_path.exists():
@@ -2832,7 +2788,8 @@ def _generate_ctrld_split_horizon_block(local_domain: str = "lan",
 
     Because the format uses indexed table sections ([network.N], [upstream.N]),
     you can't simply append a fragment — the full toml must be regenerated via
-    _build_ctrld_toml(vlan_profiles, local_resolver='127.0.0.1:5353').
+    _build_ctrld_toml(vlan_profiles).  Note: on OPNsense, Unbound handles
+    local resolution — ctrld does not need split-horizon rules at all.
 
     This function returns a plain-English example for display only.
     """
@@ -2909,8 +2866,8 @@ def save_local_hostnames(body: LocalHostnamesUpdate):
     if ctrld_cfg.get("vlan_profiles"):
         split_horizon_toml = _build_ctrld_toml(
             ctrld_cfg["vlan_profiles"],
-            local_domain=body.local_domain or "lan",
+            ctrld_port=ctrld_cfg.get("ctrld_port", 5354),
-            local_resolver=f"127.0.0.1:5353",
+            deploy_mode=ctrld_cfg.get("deploy_mode", "router"),
         )
         # Write new toml if running locally
         if ctrld_cfg.get("mode") == "local":