From abe5b4d5bbd7fe01d102d02fa049df70b0f8a036 Mon Sep 17 00:00:00 2001 From: "SREDevOps.org Agent" Date: Wed, 21 Oct 2026 15:09:51 -0300 Subject: [PATCH] Dial the sandbox pod IP for WebSocket upgrades (#10) DSH's Host fence (isTrustedApiRequest) only trusts loopback, an allow-listed host, or its own bind address, and requires any Origin to match the Host authority. The gateway forwarded the browser Host as an additional header, but websockets derives Host from the URI and cannot be overridden, so the request carried a duplicate/mismatched Host plus the browser Origin and DSH answered 403. The gateway then closed with 1011, which the DSH client retries forever: [connection] connection lost, retry #1 ... #55 Dial the pod IP instead - it is the bind address, so the derived Host passes the fence - and forward no Origin; CSWSH is already rejected by same_origin() before the upstream connection. Fall back to the service FQDN when status.podIPs is empty and bracket IPv6 literals. Reproduced against dsh-v0.2.1-rc.2: pod-IP upgrade with no Origin opens; the same upgrade carrying the browser Origin gets HTTP 403. Co-authored-by: SREDevOpsAgents --- examples/dsh-agents-as-a-service/README.md | 7 +++++++ .../gateway/gateway.py | 21 +++++++++++++++++-- .../gateway/test_gateway.py | 11 +++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/examples/dsh-agents-as-a-service/README.md b/examples/dsh-agents-as-a-service/README.md index cebff00..be2e041 100644 --- a/examples/dsh-agents-as-a-service/README.md +++ b/examples/dsh-agents-as-a-service/README.md @@ -313,6 +313,11 @@ the DSH web GUI is fully interactive. Requirements: `60-ingress.yaml` is a starting point (Traefik handles upgrades by default), - idle suspension is disabled for users with in-flight proxied requests or open WebSockets, so a long-running chat stream is not cut off mid-turn, +- the upstream upgrade dials the sandbox **pod IP** and forwards no `Host`/`Origin`: DSH's Host + fence only trusts its own bind address or a loopback/allow-listed host, and `websockets` + derives `Host` from the URI (it cannot be overridden). Forwarding the browser's origin here + makes DSH answer `403`, which shows up in the browser as an endless + `[connection] connection lost, retry #N` loop, - missing/invalid gateway auth is rejected right after the upgrade with close code `4401` (WebSocket auth has no HTTP `401` path); cross-origin or Origin-less upgrades with the cookie are rejected the same way (CSWSH protection), so clients must send a matching `Origin`; @@ -336,6 +341,8 @@ redirects `/u/` to the trailing-slash form with `308` so relative URLs res | `409` on signup | User already has a claim; use `POST /login` with the existing token or delete the user first | | Cookie set but browser keeps getting `401` | `COOKIE_SECURE=true` while serving over plain HTTP (secure cookies are not sent); use HTTPS or set `false` for local demos | | WebSocket closes immediately (`4401`) | Missing/expired gateway cookie, or the upgrade's `Origin` does not match `Host` (CSWSH check); DSH rejecting `Host` instead surfaces as close `1011` — add the host to `DSH_TRUSTED_HOSTS` | +| Console spams `[connection] connection lost, retry #N` | The upstream WS upgrade is being rejected (DSH answers `403` to the forward, surfaced as close `1011`). The gateway dials the sandbox **pod IP** and forwards no `Host`/`Origin` for upgrades; a gateway image from before that fix, or a sandbox with an empty `status.podIPs`, hits the loop | +| `manifest.webmanifest` returns `401` in the console | Browsers fetch the Web App Manifest without credentials, so the gateway's session cookie is absent and the request fails closed. The UI is unaffected; only the PWA install prompt is lost | | Signup is slow / `202` then `Waking` | Warm pool exhausted (`kubectl get sandboxes -l agents.x-k8s.io/warm-pool-sandbox`); scale the pool | | `403` from the Kubernetes API in gateway logs | The gateway ServiceAccount is missing claim/sandbox/pod-log verbs (see `50-gateway.yaml`) | | Step 3: `pod "$SB" not found` after deploying the gateway | The claim was suspended by the idle sweeper. Claims created by the gateway carry `aaas.example.com/managed-by=dsh-gateway`; hand-applied claims (steps 1–7) do not and are never swept. Upgrade the gateway to a build that honors the marker, or resume the sandbox: `kubectl -n dsh-demo patch sandbox "$SB" --type merge -p '{"spec":{"operatingMode":"Running"}}'` | diff --git a/examples/dsh-agents-as-a-service/gateway/gateway.py b/examples/dsh-agents-as-a-service/gateway/gateway.py index ba8a1a2..321e6e0 100644 --- a/examples/dsh-agents-as-a-service/gateway/gateway.py +++ b/examples/dsh-agents-as-a-service/gateway/gateway.py @@ -669,6 +669,15 @@ class Gateway: or sandbox["metadata"]["name"] ) + @staticmethod + def _pod_ip(sandbox: dict) -> str | None: + """The pod IP is the authority DSH accepts in its Host fence.""" + for entry in (sandbox.get("status") or {}).get("podIPs") or []: + ip = entry.get("ip") if isinstance(entry, dict) else entry + if ip: + return ip + return None + def _clear_handshake(self, user: str) -> None: for key in [k for k in self.state.handshake if k[0] == user]: self.state.handshake.pop(key, None) @@ -1141,7 +1150,15 @@ against the JSON API. The browser session is an HttpOnly cookie.

host = websocket.headers.get("host", "") cookie, _, _ = await self.dsh_cookie(user, sandbox, fqdn, host) path = websocket.path_params.get("path", "") - url = f"ws://{fqdn}:{self.settings.dsh_port}/{path}" + # DSH's Host fence accepts its own bind address and requires + # any Origin to have the same authority. websockets derives the + # Host header from the URI and cannot be overridden, so dial + # the pod IP instead of the service FQDN and forward no Origin; + # CSWSH is already rejected by same_origin() above. + upstream_host = self._pod_ip(sandbox) or fqdn + if ":" in upstream_host and not upstream_host.startswith("["): + upstream_host = f"[{upstream_host}]" # IPv6 literal + url = f"ws://{upstream_host}:{self.settings.dsh_port}/{path}" query = filtered_query(websocket.url.query) if query: url = f"{url}?{query}" @@ -1149,9 +1166,9 @@ against the JSON API. The browser session is an HttpOnly cookie.

key: value for key, value in websocket.headers.items() if key.lower() not in OWNED_REQUEST_HEADERS + and key.lower() != "origin" and not key.lower().startswith("sec-websocket-") } - headers["host"] = host headers["cookie"] = cookie headers["x-forwarded-host"] = host headers["x-forwarded-proto"] = websocket.url.scheme diff --git a/examples/dsh-agents-as-a-service/gateway/test_gateway.py b/examples/dsh-agents-as-a-service/gateway/test_gateway.py index 977d5e8..4e10ae8 100644 --- a/examples/dsh-agents-as-a-service/gateway/test_gateway.py +++ b/examples/dsh-agents-as-a-service/gateway/test_gateway.py @@ -38,6 +38,7 @@ from gateway import ( TOKEN = "s3cret-token" POD_UID = "pod-uid-1" +POD_IP = "10.0.0.7" FQDN = "dsh-alice-sbx.test-ns.svc.cluster.local" @@ -206,6 +207,7 @@ def add_user(k8s, user="alice", token=TOKEN, ready=True, pod_name=None, suffix=" "status": { "conditions": [{"type": "Ready", "status": "True"}] if ready else [], "serviceFQDN": FQDN, + "podIPs": [POD_IP], }, } k8s.pods[pod_name] = {"metadata": {"name": pod_name, "uid": POD_UID}} @@ -1202,13 +1204,16 @@ def test_ws_bridges_frames_and_injects_cookie(env): assert ws.receive_text() == "pong" url, kwargs = env.ws_connect.calls[0] - assert url.startswith(f"ws://{FQDN}:3080/api/remote.mux") + # DSH's Host fence: dial the pod IP (its bind address) and never forward + # the browser's Host/Origin, which would make the upgrade 403. + assert url.startswith(f"ws://{POD_IP}:3080/api/remote.mux") assert "token=secret" not in url assert "x=1" in url headers = kwargs["additional_headers"] assert "authorization" not in headers assert headers["cookie"] == "dsh-auth-a=one" - assert headers["host"] == "testserver" + assert "host" not in headers + assert "origin" not in headers assert env.ws_connect.instances[0].sent == ["hello"] @@ -1228,7 +1233,7 @@ def test_ws_cookie_same_origin_bridges_frames(env): headers = env.ws_connect.calls[0][1]["additional_headers"] assert headers["cookie"] == "dsh-auth-a=one" - assert headers["origin"] == "http://testserver" + assert "origin" not in headers def test_ws_handshake_failure_closes_1011(env):