From 6301c21beafaccd78f569ed1c1726b70ae150fc7 Mon Sep 17 00:00:00 2001 From: "SREDevOps.org Agent" Date: Wed, 21 Oct 2026 12:10:03 -0300 Subject: [PATCH] Make DSH agents-as-a-service browser-usable; fix probe crash-loop (#3) * Rewrite DSH gateway for browser sessions, HTTP and WebSocket - Replace the Flask proxy with a Starlette ASGI app (uvicorn factory), keeping the control plane (claims, wake/suspend, idle sweeper). - Add HttpOnly gateway sessions and a server-side DSH launch-token handshake: read the token from pod logs, exchange it for a dsh-auth cookie, cache per (user, pod UID, Host), refresh once on 401. - Proxy HTTP/SSE and WebSocket (/api/remote.mux) under /u//, with header stripping/injection, Location rewriting and wake-on-connect. - Add a landing/signup/login page, health endpoint, pinned dependencies, a read-only-rootfs gateway image, 87 cluster-free tests and docs. * Fix DSH probe crash-loop and harden gateway manifests - Probe /favicon.svg instead of /: DSH gates / behind its launch-token cookie and returns 401, which made warm-pool pods crash-loop. - Pass DSH_TRUSTED_HOSTS through the entrypoint as --trusted-host args for public/ingress hosts; keep localhost trusted. - Deploy the published :gateway image, add pods/pods-log RBAC, /healthz probes, read-only rootfs with a /tmp emptyDir and non-root security. - Add optional Traefik Ingress and document the prerequisites. * Update CI, smoke tests, build tooling and docs - Run gateway pytest/ruff and build/smoke-test the image in CI. - Parameterize the DSH build version and pnpm version inputs. - Exercise cookie signup, HTML proxy, WebSocket auth, suspend/wake and delete in run-test-kind.sh. - Add Makefile/build.sh targets and document the architecture, handshake, probes, trusted hosts, troubleshooting and limitations. --------- Co-authored-by: SREDevOpsAgents --- .env.sample | 25 + .github/workflows/multi-build-gateway.yaml | 10 +- .github/workflows/multi-build.yaml | 26 +- .github/workflows/validate.yaml | 80 +- .gitignore | 12 +- Dockerfile | 13 + Makefile | 37 + README.md | 144 +- build.sh | 75 +- deploy/k8s/sandbox-template.yaml | 30 +- docs/architecture.md | 299 ++++ .../10-sandboxtemplate.yaml | 23 +- .../dsh-agents-as-a-service/50-gateway.yaml | 80 +- .../dsh-agents-as-a-service/60-ingress.yaml | 46 + examples/dsh-agents-as-a-service/README.md | 203 ++- .../gateway/Dockerfile | 13 +- .../dsh-agents-as-a-service/gateway/README.md | 184 ++ .../gateway/gateway.py | 1498 +++++++++++++---- .../gateway/pyproject.toml | 14 + .../gateway/requirements-dev.txt | 4 + .../gateway/requirements.txt | 9 +- .../gateway/test_gateway.py | 1378 ++++++++++++--- .../dsh-agents-as-a-service/run-test-kind.sh | 157 +- 23 files changed, 3700 insertions(+), 660 deletions(-) create mode 100644 Makefile mode change 100644 => 100755 build.sh create mode 100644 docs/architecture.md create mode 100644 examples/dsh-agents-as-a-service/60-ingress.yaml create mode 100644 examples/dsh-agents-as-a-service/gateway/README.md create mode 100644 examples/dsh-agents-as-a-service/gateway/pyproject.toml create mode 100644 examples/dsh-agents-as-a-service/gateway/requirements-dev.txt diff --git a/.env.sample b/.env.sample index c53f389..2f919ab 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,29 @@ +# Copy to .env and fill in. Never commit a real key. + +# --- DSH Web image (Dockerfile / build.sh) --- DEEPSEEK_API_KEY= DSH_VERSION=dsh-v0.2.1-rc.2 PNPM_VERSION=11.28.5 +# DSH Web port inside the container, and the local port published by +# build.sh / docker run. The gateway also uses it as the upstream port. DSH_PORT=3080 +# Comma/space separated extra Host allow-list entries for DSH (for an +# Ingress/public hostname). localhost and 127.0.0.1 are always trusted. +DSH_TRUSTED_HOSTS= + +# --- Gateway (examples/dsh-agents-as-a-service) --- +GATEWAY_IMAGE=ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway +NAMESPACE=dsh-demo +POOL=dsh-pool +# Seconds a sandbox may stay idle before the sweeper suspends it. +IDLE_TIMEOUT=60 +# Seconds a waking request may wait for the sandbox to become Ready. +WAKE_TIMEOUT=120 +DSH_CONTAINER=dsh-web +DSH_LOG_TAIL_LINES=2000 +COOKIE_NAME=dsh_gw_session +# Set to true only behind HTTPS; secure cookies are not sent over plain HTTP. +COOKIE_SECURE=false +COOKIE_MAX_AGE=604800 +MAX_BODY_MB=32 +HTTP_TIMEOUT=300 diff --git a/.github/workflows/multi-build-gateway.yaml b/.github/workflows/multi-build-gateway.yaml index d103622..261d525 100644 --- a/.github/workflows/multi-build-gateway.yaml +++ b/.github/workflows/multi-build-gateway.yaml @@ -1,5 +1,7 @@ -# This workflow builds a multi-arch Docker image using GitHub Actions and separated Github Runners with native support for ARM64 and AMD64 architectures, without using QEMU emulation. -# It uses Docker Buildx to build and push the image to GitHub Container Registry (GHCR). +# This workflow builds the multi-arch gateway image (Starlette ASGI, served by +# uvicorn on :8080) and pushes it to GHCR as +# ghcr.io//:gateway. The DSH Web image is built by multi-build.yaml. +# It uses Docker Buildx with native AMD64 and ARM64 runners, without QEMU. name: Gateway Build multi arch Docker Image with separate Github Runners on: @@ -11,9 +13,7 @@ on: - 'examples/dsh-agents-as-a-service/gateway/**' - '.github/workflows/multi-build-gateway.yaml' env: - # The name of the Docker image to be built and pushed to GHCR - # The image name is derived from the GitHub repository name and the GitHub Container Registry (GHCR) URL. - # The image name will be in the format: ghcr.io// + # ghcr.io//; this workflow always publishes the `:gateway` tag. GHCR_IMAGE: ghcr.io/${{ github.repository }} permissions: # Global permissions for the workflow, which can be overridden at the job level diff --git a/.github/workflows/multi-build.yaml b/.github/workflows/multi-build.yaml index dc7dd5f..a0638f1 100644 --- a/.github/workflows/multi-build.yaml +++ b/.github/workflows/multi-build.yaml @@ -4,6 +4,17 @@ name: Build multi arch Docker Image with separate Github Runners on: workflow_dispatch: + inputs: + dsh_version: + description: "DSH git ref to build (tag or branch)" + required: true + default: dsh-v0.2.1-rc.2 + type: string + pnpm_version: + description: "pnpm version used to build DSH" + required: true + default: "11.28.5" + type: string push: branches: - main @@ -15,7 +26,10 @@ env: # The image name is derived from the GitHub repository name and the GitHub Container Registry (GHCR) URL. # The image name will be in the format: ghcr.io// GHCR_IMAGE: ghcr.io/${{ github.repository }} - DSH_VERSION: dsh-v0.2.1-rc.2 #TODO: Make this configurable from workflow inputs and/or an custom Github Action + # workflow_dispatch inputs win; pushes to main fall back to the pinned + # defaults. Both values flow into the Docker build args and the image tags. + DSH_VERSION: ${{ inputs.dsh_version || 'dsh-v0.2.1-rc.2' }} + PNPM_VERSION: ${{ inputs.pnpm_version || '11.28.5' }} permissions: # Global permissions for the workflow, which can be overridden at the job level contents: read @@ -130,14 +144,14 @@ jobs: uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 env: DOCKER_BUILDKIT: 1 - DSH_VERSION: dsh-v0.2.1-rc.2 #TODO: Make this configurable from workflow inputs and/or an custom Github Action - PNPM_VERSION: 11.28.5 DSH_PORT: 3080 with: context: . platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} - build-args: DSH_VERSION=${{ env.DSH_VERSION }} + build-args: | + DSH_VERSION=${{ env.DSH_VERSION }} + PNPM_VERSION=${{ env.PNPM_VERSION }} annotations: ${{ steps.meta.outputs.annotations }} outputs: type=image,name=${{ env.GHCR_IMAGE }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true cache-from: type=gha,scope=${{ github.repository }}-${{ github.ref_name }}-${{ matrix.platform }} @@ -209,8 +223,6 @@ jobs: # The metadata includes information such as the image name, tags, and labels. id: meta uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - env: - DSH_VERSION: ${{ env.DSH_VERSION }} with: images: ${{ env.GHCR_IMAGE }} annotations: | @@ -218,7 +230,7 @@ jobs: tags: | type=raw,value=main,enable=${{ github.ref_name == 'main' }} type=raw,value=latest,enable=${{ github.ref_name == 'main' }} - type=raw,value=${{ env.DSH_VERSION }},enable=${{ github.ref_name == 'main' }} + type=raw,value=${{ env.DSH_VERSION }},enable=${{ github.ref_name == 'main' || github.event_name == 'workflow_dispatch' }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index ba9f26f..f9d2520 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -1,8 +1,9 @@ name: Validate # Validates the Kubernetes manifests against the real agent-sandbox CRD -# schema (server-side dry-run on a throwaway kind cluster) and runs the -# gateway unit tests. Runs on every PR and on pushes to main. +# schema (server-side dry-run on a throwaway kind cluster), runs the gateway +# unit tests + lint, and builds/smoke-tests the gateway image without a +# cluster. Runs on every PR and on pushes to main. on: pull_request: push: @@ -45,6 +46,10 @@ jobs: echo "all manifests validated" gateway-tests: + # The gateway lives in examples/dsh-agents-as-a-service/gateway/. Its + # runtime pins and dev pins are kept separate so the image only ships + # requirements.txt while CI installs both files. + name: Gateway unit tests and lint runs-on: ubuntu-latest defaults: run: @@ -57,9 +62,76 @@ jobs: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" + cache: pip + cache-dependency-path: | + examples/dsh-agents-as-a-service/gateway/requirements.txt + examples/dsh-agents-as-a-service/gateway/requirements-dev.txt - name: Install dependencies - run: pip install -r requirements.txt pytest + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + + - name: Lint (ruff) + run: python -m ruff check . - name: Run gateway unit tests - run: python -m pytest test_gateway.py -v + run: python -m pytest -v + + gateway-image: + # No cluster and no real credentials: the gateway must start and answer + # /healthz with a dummy kubeconfig, which catches broken imports, missing + # runtime deps, and an unresponsive ASGI entrypoint. + name: Build and smoke-test gateway image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build gateway image + run: | + docker build \ + -t dsh-aaas-gateway:ci \ + examples/dsh-agents-as-a-service/gateway/ + + - name: Smoke-test /healthz without a cluster + run: | + set -euo pipefail + cat > /tmp/kubeconfig <<'EOF' + apiVersion: v1 + kind: Config + clusters: + - name: dummy + cluster: + server: https://127.0.0.1:6443 + insecure-skip-tls-verify: true + contexts: + - name: dummy + context: + cluster: dummy + user: dummy + users: + - name: dummy + user: {} + current-context: dummy + EOF + docker run -d --name gateway-smoke \ + -p 18080:8080 \ + -e NAMESPACE=dsh-demo \ + -e POOL=dsh-pool \ + -e KUBECONFIG=/tmp/kubeconfig \ + -v /tmp/kubeconfig:/tmp/kubeconfig:ro \ + dsh-aaas-gateway:ci + trap 'docker logs gateway-smoke || true; docker rm -f gateway-smoke || true' EXIT + + code="" + for _ in $(seq 1 30); do + code=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:18080/healthz || true) + [ "$code" = "200" ] && break + sleep 1 + done + if [ "$code" != "200" ]; then + echo "gateway /healthz returned '$code' instead of 200" >&2 + exit 1 + fi + echo "gateway image healthy: GET /healthz -> 200" diff --git a/.gitignore b/.gitignore index a80a91a..1cec806 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,7 @@ web_modules/ # dotenv environment variable files .env .env.* -!.env.example +!.env.sample # parcel-bundler cache (https://parceljs.org/) .cache @@ -148,8 +148,14 @@ vite.config.ts.timestamp-* # Turbo Repo .turbo -!.env.sample - # Python __pycache__/ *.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +.coverage.* +htmlcov/ +.venv/ +venv/ diff --git a/Dockerfile b/Dockerfile index 95ff466..2de7da6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ set -eu : "${DSH_PORT:=3080}" : "${DSH_HOST:=}" +: "${DSH_TRUSTED_HOSTS:=}" DSH_PUBLIC_URL="${DSH_PUBLIC_URL-http://localhost:${DSH_PORT}}" # DSH rejects wildcard hosts. Pick the container/pod's first non-loopback IPv4. @@ -87,6 +88,18 @@ if [ -n "$DSH_PUBLIC_URL" ]; then set -- --public-url "$DSH_PUBLIC_URL" "$@" fi +# Additional trusted hosts (e.g. the public gateway host behind an Ingress). +# DSH Host/Origin-fences /api: without this, requests forwarded with the +# browser's Host header are rejected. Accepts comma- or space-separated names; +# localhost/127.0.0.1 are always trusted below. +old_ifs="$IFS" +IFS=', ' +for trusted_host in $DSH_TRUSTED_HOSTS; do + [ -n "$trusted_host" ] || continue + set -- "$@" --trusted-host "$trusted_host" +done +IFS="$old_ifs" + exec node /app/lib/bin.js web \ --no-open \ --host "$DSH_HOST" \ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..da1d75d --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +# Local build and test helpers. Every variable can be overridden, e.g. +# make image DSH_VERSION=dsh-v0.2.1-rc.2 IMAGE=dsh-web:dev +# make test +DSH_VERSION ?= dsh-v0.2.1-rc.2 +PNPM_VERSION ?= 11.28.5 +IMAGE ?= dsh-web:0.2.1-rc.2 +GATEWAY_IMAGE ?= ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway +GATEWAY_DIR := examples/dsh-agents-as-a-service/gateway +PYTHON ?= python3 +DSH_PORT ?= 3080 + +.PHONY: help image gateway run test lint + +help: ## Show available targets + @grep -E '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | sed 's/:.*## /\t/' + +image: ## Build the DSH Web image + docker build \ + --build-arg DSH_VERSION=$(DSH_VERSION) \ + --build-arg PNPM_VERSION=$(PNPM_VERSION) \ + -t $(IMAGE) . + +gateway: ## Build the gateway image + docker build -t $(GATEWAY_IMAGE) $(GATEWAY_DIR)/ + +run: image ## Build and run the DSH Web image + docker run --rm -it \ + -p $(DSH_PORT):3080 \ + -v dsh-home:/home/node/.dsh \ + -e DEEPSEEK_API_KEY \ + $(IMAGE) + +test: ## Run the gateway unit tests (needs requirements + requirements-dev installed) + cd $(GATEWAY_DIR) && $(PYTHON) -m pytest -q + +lint: ## Lint the gateway with ruff + cd $(GATEWAY_DIR) && ruff check . diff --git a/README.md b/README.md index ce098d7..d4c9bd7 100644 --- a/README.md +++ b/README.md @@ -3,34 +3,110 @@ Packages [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) Web (`dsh web`) into a multi-arch, Kubernetes-ready container image, and ships the [agents-as-a-service](examples/dsh-agents-as-a-service/) platform pattern — running a personal -DSH Web instance per user on [kubernetes-sigs/agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox). +DSH Web instance per user on [kubernetes-sigs/agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) +behind a small browser-usable gateway. ## What's in this repo | Path | Purpose | | --- | --- | -| [`Dockerfile`](Dockerfile) | Multi-stage build that clones DSH, `pnpm pack`s the CLI, and produces a slim runtime image | -| [`build.sh`](build.sh) | Local build + run (builds `dsh-web:` and runs it) | -| [`.github/workflows/multi-build.yaml`](.github/workflows/multi-build.yaml) | CI build (amd64 + arm64) and push to GHCR | -| [`deploy/k8s/sandbox-template.yaml`](deploy/k8s/sandbox-template.yaml) | Minimal single-tenant `SandboxTemplate` | +| [`Dockerfile`](Dockerfile) | Multi-stage build that clones DSH, `pnpm pack`s the CLI, and produces a slim runtime image (`:3080`) | +| [`build.sh`](build.sh), [`Makefile`](Makefile) | Local image builds: DSH Web and gateway, plus test/lint targets | | [`examples/dsh-agents-as-a-service/`](examples/dsh-agents-as-a-service/) | The full multi-tenant platform pattern (warm pool, per-user claims, gateway, smoke test) | +| [`examples/dsh-agents-as-a-service/gateway/`](examples/dsh-agents-as-a-service/gateway/) | Starlette ASGI gateway: signup, cookie sessions, `/u/{user}/**` HTTP + WebSocket proxy, idle sweeper | +| [`deploy/k8s/sandbox-template.yaml`](deploy/k8s/sandbox-template.yaml) | Minimal single-tenant `SandboxTemplate` | +| [`docs/architecture.md`](docs/architecture.md) | Container and sequence diagrams, design decisions, security notes | +| [`.github/workflows/`](.github/workflows/) | CI: manifests dry-run, gateway tests/lint, multi-arch image builds | -## Quick start (image) +## Published images -The image is published to GitHub Container Registry as -`ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:`, where `` is `main`, -`latest`, or a DSH version tag such as `dsh-v0.2.1-rc.2`. +Everything is published to GitHub Container Registry as +`ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox`: + +| Tag | Contents | Port | +| --- | --- | --- | +| `dsh-v0.2.1-rc.2` (also `main`, `latest`) | DSH Web image | `3080` | +| `gateway` | Starlette gateway image | `8080` | + +The DSH Web tag matches the upstream DSH git tag it was built from. + +## Quick start (DSH Web image) ```bash -docker run --rm -it \ +docker run -d --name dsh-web \ -p 3080:3080 \ -v dsh-home:/home/node/.dsh \ -e DEEPSEEK_API_KEY \ ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.2 ``` -> **Local port note:** `dsh web` listens on `3080` inside the container. If `3080` is -> already in use on your workstation, map a different host port (e.g. `-p 18081:3080`). +**DSH Web is not usable at a bare `/`.** Without its launch-token cookie it answers `401`, and +the token is only printed in the startup logs: + +```bash +docker logs dsh-web 2>&1 | grep -m1 '?token=' +# dsh web: http://localhost:3080/?token=... +``` + +Open the printed `?token=...` URL once — DSH sets its auth cookie and the UI loads normally +afterwards. Hitting `/` directly (or after the cookie expires) returns `401` by design; that is +why Kubernetes probes use `/favicon.svg` (see below) and why the gateway performs this +handshake for the browser. + +> **Different host port?** If `3080` is already taken on your workstation, map another port and +> set the public URL so the printed token URL matches: `-p 18081:3080 -e +> DSH_PUBLIC_URL=http://localhost:18081`, then open `http://localhost:18081/?token=...`. + +## Quick start (Kubernetes) + +```bash +kubectl create namespace dsh-demo --dry-run=client -o yaml | kubectl apply -f - +kubectl -n dsh-demo create secret generic dsh-provider-keys \ + --from-literal=DEEPSEEK_API_KEY="sk-..." +kubectl apply -f deploy/k8s/sandbox-template.yaml + +# A template needs a warm pool and a claim to become a running Sandbox. +kubectl apply -f - <<'EOF' +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxWarmPool +metadata: + name: dsh-pool + namespace: dsh-demo +spec: + replicas: 1 + sandboxTemplateRef: + name: dsh-web-template +--- +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxClaim +metadata: + name: dsh + namespace: dsh-demo +spec: + warmPoolRef: + name: dsh-pool +EOF +kubectl -n dsh-demo wait sandboxclaim/dsh --for=condition=Ready --timeout=180s +``` + +Then reach the instance through its launch-token URL: + +```bash +SB=$(kubectl -n dsh-demo get sandboxclaim dsh -o jsonpath='{.status.sandbox.name}') +kubectl -n dsh-demo logs "$SB" | grep -m1 '?token=' +kubectl -n dsh-demo port-forward "pod/$SB" 18081:3080 +# open http://localhost:18081/?token=... from the log line +``` + +Two things bite first-time setups: + +- **Probes must not hit `/`.** DSH returns `401` there without the launch token, so a + readiness/liveness probe on `/` crash-loops the pod. The shipped templates use + `/favicon.svg`, which is unauthenticated; keep that path in custom templates. +- **Host allow-listing.** DSH rejects unknown `Host` headers. `localhost` and `127.0.0.1` are + always trusted; for any other hostname add it to the template's `DSH_TRUSTED_HOSTS` (comma or + space separated). The gateway forwards the browser's `Host`, so the public gateway hostname + must be listed there. ## The multi-user platform pattern @@ -38,15 +114,23 @@ docker run --rm -it \ [hermes-agents-as-a-service](https://github.com/kubernetes-sigs/agent-sandbox/tree/main/examples/hermes-agents-as-a-service) example to DSH Web. It demonstrates the platform shape — one `SandboxTemplate`, one warm pool, one tiny `SandboxClaim` per user, and `operatingMode` as the cost dial — plus a gateway that -turns it into a usable service (signup, per-user tokens, `/u/{user}/**` proxy with -wake-on-connect, idle suspend). See its [README](examples/dsh-agents-as-a-service/README.md) for -the full walkthrough. +turns it into a usable service. + +**Browser flow:** open the gateway landing page, sign up, and get an `HttpOnly` session cookie. +The browser then browses `http:///u//`, which proxies HTTP **and WebSocket** to +that user's DSH Web. The gateway reads the DSH launch token from the pod logs, performs the +token handshake server-side, and injects the resulting DSH cookie on every proxied request — the +browser never sees the token. Suspended instances are woken transparently on the next request; +idle ones are suspended again after `IDLE_TIMEOUT`. + +See the [example README](examples/dsh-agents-as-a-service/README.md) for the full walkthrough +and [docs/architecture.md](docs/architecture.md) for the diagrams and trade-offs. ### Mechanism mapping: Hermes → DSH | Hermes example mechanism | Role | Adaptation for DSH | | :--- | :--- | :--- | -| `SandboxTemplate` | Reusable Pod blueprint | Image `dsh-web`, port `3080` | +| `SandboxTemplate` | Reusable Pod blueprint | Image `dsh-web`, port `3080`, probes on `/favicon.svg` | | `volumeClaimTemplates` | Auto-create a PVC per user | Mount `/home/node/.dsh` (DSH state) | | `SandboxWarmPool` | Pre-warmed instances | Keep as-is; size for expected concurrency | | `SandboxClaim` + `warmPoolRef` | "Registration" adopts a pooled instance | Keep exactly as-is; one claim per user | @@ -60,19 +144,37 @@ the full walkthrough. - **Single port** — DSH Web serves everything on `3080`; no API/dashboard port split. - **One secret** — only `DEEPSEEK_API_KEY`; no dashboard/session/api-server credentials. - **Host forwarding** — the gateway forwards `Host` + `X-Forwarded-*` (and keeps - `DSH_PUBLIC_URL` empty) so DSH generates reachable links. -- **WebSocket caveat** — the requests-based proxy handles SSE but not WebSocket upgrades; use a - path-based Ingress per user for full web-GUI interactivity. + `DSH_PUBLIC_URL` empty) so DSH generates reachable links; add the public hostname to + `DSH_TRUSTED_HOSTS`. +- **Browser sessions + WebSocket** — the gateway is Starlette ASGI, so it proxies WebSocket + upgrades as well as HTTP/SSE, and authenticates browsers with an `HttpOnly` cookie instead of + exposing per-user bearer tokens in URLs. ## Build it yourself ```bash -./build.sh # docker build + run (dsh-web:0.2.1-rc.2) -# or +./build.sh # build the DSH Web image, then run it interactively +./build.sh web # build the DSH Web image only +./build.sh gateway # build the gateway image + +make image # same as ./build.sh web +make gateway # gateway image +make test # gateway unit tests (pytest) +make lint # gateway lint (ruff) +``` + +Or directly: + +```bash docker build \ --build-arg DSH_VERSION=dsh-v0.2.1-rc.2 \ + --build-arg PNPM_VERSION=11.28.5 \ -t dsh-web:0.2.1-rc.2 . + +docker build -t ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway \ + examples/dsh-agents-as-a-service/gateway/ ``` See [`Dockerfile`](Dockerfile) for build args (`DSH_REPO`, `DSH_VERSION`, `PNPM_VERSION`) and the runtime image contract (non-root `node` user, `DSH_HOME=/home/node/.dsh`, `EXPOSE 3080`). +`.env.sample` documents the runtime variables for both images. diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 index 02678d4..30572ff --- a/build.sh +++ b/build.sh @@ -1,9 +1,68 @@ -docker build \ - --build-arg DSH_VERSION=dsh-v0.2.1-rc.2 \ - -t dsh-web:0.2.1-rc.2 . +#!/usr/bin/env bash +# Build the DSH Web and/or gateway images locally. +# +# Usage: +# ./build.sh # build the DSH Web image, then run it interactively +# ./build.sh web # build the DSH Web image only +# ./build.sh gateway # build the gateway image (examples/.../gateway/) +# +# Overridable environment: +# DSH_VERSION DSH git ref to build (default dsh-v0.2.1-rc.2) +# PNPM_VERSION pnpm version (default 11.28.5) +# IMAGE DSH Web image tag (default dsh-web:) +# DSH_PORT host port for docker run (default 3080) +# GATEWAY_IMAGE gateway image tag (default ghcr.io/...:gateway) +# RUN 0 to build without running (default 1) +set -euo pipefail +cd "$(dirname "$0")" -docker run --rm -it \ - -p 3080:3080 \ - -v dsh-home:/home/node/.dsh \ - -e DEEPSEEK_API_KEY \ - dsh-web:0.2.1-rc.2 \ No newline at end of file +DSH_VERSION="${DSH_VERSION:-dsh-v0.2.1-rc.2}" +PNPM_VERSION="${PNPM_VERSION:-11.28.5}" +IMAGE="${IMAGE:-dsh-web:${DSH_VERSION#dsh-v}}" +GATEWAY_IMAGE="${GATEWAY_IMAGE:-ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway}" +DSH_PORT="${DSH_PORT:-3080}" +RUN="${RUN:-1}" +GATEWAY_DIR=examples/dsh-agents-as-a-service/gateway + +build_web() { + docker build \ + --build-arg "DSH_VERSION=${DSH_VERSION}" \ + --build-arg "PNPM_VERSION=${PNPM_VERSION}" \ + -t "${IMAGE}" \ + "$@" \ + . +} + +build_gateway() { + docker build -t "${GATEWAY_IMAGE}" "$@" "${GATEWAY_DIR}/" +} + +usage() { + echo "usage: $0 [run|web|gateway] [extra docker build args...]" >&2 + exit 2 +} + +case "${1:-run}" in + run) + shift + build_web "$@" + if [ "${RUN}" = "1" ]; then + docker run --rm -it \ + -p "${DSH_PORT}:3080" \ + -v dsh-home:/home/node/.dsh \ + -e DEEPSEEK_API_KEY \ + "${IMAGE}" + fi + ;; + web) + shift + build_web "$@" + ;; + gateway) + shift + build_gateway "$@" + ;; + *) + usage + ;; +esac diff --git a/deploy/k8s/sandbox-template.yaml b/deploy/k8s/sandbox-template.yaml index b45a1c4..6b63433 100644 --- a/deploy/k8s/sandbox-template.yaml +++ b/deploy/k8s/sandbox-template.yaml @@ -3,6 +3,13 @@ # This is the "just one DSH Web" quick-start template. For the full # multi-tenant platform (warm pool, per-user claims, gateway, idle-suspend), # see examples/dsh-agents-as-a-service/. +# +# Prerequisites (create these first, or apply 00-prereqs.yaml from the +# examples directory): +# - Namespace dsh-demo. +# - Secret dsh-provider-keys in dsh-demo with key DEEPSEEK_API_KEY +# (injected via envFrom; the pod starts without it, only model calls fail). +# The agent-sandbox controller (v1.0.7, API v1beta1) must be installed. apiVersion: extensions.agents.x-k8s.io/v1beta1 kind: SandboxTemplate metadata: @@ -27,21 +34,40 @@ spec: envFrom: - secretRef: name: dsh-provider-keys # Secret with key DEEPSEEK_API_KEY + env: + # Empty => DSH derives links from the request Host. Keep it empty + # when fronting the sandbox with a proxy/gateway. + - name: DSH_PUBLIC_URL + value: "" + # DSH Host/Origin-fences /api. Empty trusts only localhost, which + # is right for `kubectl port-forward`. Set this to the public host + # (e.g. dsh.example.com) if you expose DSH behind an Ingress, so + # requests forwarded with the browser's Host are accepted. + - name: DSH_TRUSTED_HOSTS + value: "" volumeMounts: - name: dsh-home mountPath: /home/node/.dsh # DSH_HOME + # / is behind DSH's launch-token cookie (401 without it); use an + # unauthenticated path for probes. readinessProbe: httpGet: - path: / + path: /favicon.svg port: 3080 initialDelaySeconds: 5 periodSeconds: 5 livenessProbe: httpGet: - path: / + path: /favicon.svg port: 3080 initialDelaySeconds: 15 periodSeconds: 15 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault volumeClaimTemplates: - metadata: name: dsh-home diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8049b3c --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,299 @@ +# Architecture + +This document describes the browser-usable **agents-as-a-service** platform in +[`examples/dsh-agents-as-a-service/`](../examples/dsh-agents-as-a-service/): how the gateway, +Kubernetes, and DSH Web fit together, and why the design looks the way it does. + +Scope: one Kubernetes namespace (`dsh-demo`), one shared DSH Web image, one gateway deployment, +and one DSH Web pod + PVC per user. The cluster is assumed to run +[agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) v1.0.7 with the extensions +(`SandboxTemplate`, `SandboxWarmPool`, `SandboxClaim`). + +## Container view + +```mermaid +flowchart LR + browser["Browser"] + + subgraph cluster["Kubernetes cluster - namespace dsh-demo"] + gw["Gateway Deployment
Starlette ASGI on uvicorn :8080
landing page + auth + proxy + sweeper"] + api["Kubernetes API Server"] + ctrl["agent-sandbox controller"] + pool["SandboxWarmPool dsh-pool
2 pre-warmed spares"] + claim["SandboxClaim dsh-USER
one per user"] + sb["Sandbox dsh-pool-xxxx
spec.operatingMode"] + svc["Headless Service
status.serviceFQDN"] + pod["DSH Web pod
node process :3080"] + pvc[("PVC dsh-home-SANDBOX
/home/node/.dsh")] + end + + browser -->|"HTTP + WebSocket
/u/USER/**"| gw + gw -->|"claims, sandboxes, pods, pod logs"| api + api --> ctrl + ctrl -->|"adopt / suspend / resume"| claim + claim --> sb + pool -.->|"replenishes"| sb + sb --> svc + svc --> pod + pod --> pvc +``` + +Responsibilities: + +| Component | Responsibility | +| --- | --- | +| **Browser** | Holds only the gateway session cookie (`dsh_gw_session`). Talks exclusively to the gateway. | +| **Gateway** | Signup/login, session auth, `SandboxClaim` lifecycle, wake-on-connect, idle sweeper, HTTP/SSE/WebSocket reverse proxy, DSH launch-token handshake. | +| **Kubernetes API** | Source of truth for claims, sandboxes, and pods; also the log source for the DSH launch token. | +| **agent-sandbox controller** | Adopts warm spares on claim, flips `operatingMode` (Running/Suspended), garbage-collects sandbox + PVC on claim delete. | +| **DSH Web pod** | One `dsh web` process per user, state on the per-user PVC, listening on `3080`. | +| **Warm pool** | Keeps `N` Ready sandboxes so signup is a label flip instead of a cold start. | + +## Flows + +### Signup (browser) + +```mermaid +sequenceDiagram + autonumber + actor B as Browser + participant G as Gateway + participant K as Kubernetes API + participant C as agent-sandbox controller + + B->>G: POST /users (form: user=bob, Origin) + G->>G: validate user name and body size + G->>K: create SandboxClaim dsh-bob (warmPoolRef) + K->>C: reconcile claim + C-->>K: adopt spare, Ready, status.sandbox.name = dsh-pool-xxxx + G->>K: read claim (status) + sandbox + pod + G-->>B: 303 Location /u/bob/ + Set-Cookie dsh_gw_session + Note over G: JSON clients instead get 201 (Ready),
202 (accepted, still waking), or 409 (conflict) +``` + +Form signup waits up to `WAKE_TIMEOUT` for the claim to become `Ready`, then redirects; the +first proxied request also waits for `Ready` and performs the DSH handshake (below). A `409` +means the user already has a claim, in which case `POST /login` (bearer token) or the existing +cookie is the right path. + +### First request / proxy (with handshake) + +```mermaid +sequenceDiagram + autonumber + actor B as Browser + participant G as Gateway + participant K as Kubernetes API + participant D as DSH Web pod + + B->>G: GET /u/bob/ (Cookie dsh_gw_session) + G->>G: authenticate cookie, user = bob + G->>K: resolve claim -> sandbox -> pod + alt sandbox Suspended + G->>K: patch operatingMode=Running (wake) + loop until Ready or WAKE_TIMEOUT + G->>K: read sandbox Ready condition + end + end + G->>K: read pod logs (tail) + K-->>G: "dsh web: ...?token=..." line + G->>D: GET /?token=... (browser Host, X-Forwarded-*) + D-->>G: 303 + Set-Cookie dsh-auth-* + G->>G: cache cookie for (user, pod UID, Host) + G->>G: strip Cookie/Authorization/hop-by-hop headers + G->>D: replay request with dsh-auth cookie + D-->>G: 200 HTML / SSE stream + G-->>B: 200 (upstream Set-Cookie dropped, Location rewritten) +``` + +Handshake cache behavior: one entry per `(user, pod UID, Host)`; refreshed once when DSH answers +`401`; dropped when the sandbox is suspended or deleted. The launch token and the DSH auth +cookie never reach the browser. + +### Wake-on-connect and idle suspend + +```mermaid +sequenceDiagram + autonumber + participant S as Sweeper - lifespan task + participant G as Gateway + participant K as Kubernetes API + actor B as Browser + + S->>G: no proxied request for IDLE_TIMEOUT + G->>K: patch sandbox operatingMode=Suspended + K-->>G: pod deleted, PVC + Service retained + Note over S,G: in-flight requests/WebSockets are tracked,
so a busy turn is never suspended + B->>G: GET /u/bob/ after a while + G->>K: patch operatingMode=Running + G->>K: wait for Ready (WAKE_TIMEOUT) + G-->>B: request held, then served +``` + +Suspend deletes only the pod. The PVC keeps conversations, sessions, and config, so resume +reattaches exactly the same state. + +### WebSocket + +```mermaid +sequenceDiagram + autonumber + actor B as Browser + participant G as Gateway + participant D as DSH Web pod + + B->>G: GET /u/bob/... Upgrade: websocket (Cookie dsh_gw_session) + G->>G: authenticate + mark user in-flight + alt cookie missing/invalid + G-->>B: 101 upgrade, then close 4401 + else authenticated + G->>D: websockets.connect(path, Host, Cookie dsh-auth) + alt upstream handshake fails + G-->>B: 101 upgrade, then close 1011 + else bridged + G-->>B: 101 Switching Protocols + loop frames + B->>D: frames proxied through G + D->>B: frames proxied through G + end + G->>G: clear in-flight on close + end + end +``` + +## Decisions and trade-offs + +### ADR-1: Starlette ASGI instead of a sync Flask proxy + +- **Decision:** implement the gateway as a Starlette app served by `uvicorn + gateway:create_app --factory`. +- **Why:** the browser needs WebSocket upgrades, streaming SSE responses, and concurrent + long-poll wake requests. A requests-based Flask proxy cannot upgrade connections and blocks + workers while a sandbox wakes. +- **Cost:** async code paths and explicit bridging for the sync `kubernetes` client (offloaded + to threads). Accepted: one small service, one framework. + +### ADR-2: Path-based routing (`/u/{user}/`) instead of per-user subdomains + +- **Decision:** serve each user under `https://gateway/u/{user}/`. +- **Why:** one hostname, one TLS certificate, no wildcard DNS or per-user Ingress objects. DSH + uses relative asset/API URLs and injects ``, so a path prefix works; the + gateway rewrites absolute-path `Location` headers and redirects `/u/{user}` to + `/u/{user}/`. +- **Cost:** any upstream code that emits root-absolute URLs would need rewriting; cookie + domains are shared (mitigated by stripping DSH `Set-Cookie`). If DSH ever changes its base + handling, per-user subdomains are the fallback. + +### ADR-3: Server-side launch-token handshake instead of exposing the token + +- **Decision:** the gateway reads DSH's launch token from pod logs, performs the token + handshake itself, caches DSH's auth cookie, and authenticates the browser with its own + `HttpOnly` cookie. +- **Why:** DSH has no token API and returns `401` on `/` without the launch-token cookie. + Handing the token to the browser would put it in URLs, history, and logs, and would let users + bypass gateway auth. +- **Cost:** log reading permission and a brittle parser; see [Upstream feature + request](#upstream-feature-request-remove-log-scraping). This is the least durable part of the + design and the first thing to replace. + +### ADR-4: In-memory state, single replica + +- **Decision:** idle clocks, handshake cache, and in-flight tracking live in gateway memory; + the Deployment runs one replica. +- **Why:** the simplest thing that works, with no extra datastore for a demo/reference + platform. +- **Cost:** a restart forgets idle timers (sandboxes stay Running until their next request) and + the handshake cache; two replicas would double-suspend and race. Scaling out requires moving + this state to a shared store (Redis/Postgres) or leader election. Documented instead of + hidden. + +### ADR-5: Claims as the user record + +- **Decision:** a user is a `SandboxClaim` named `dsh-`; the bearer-token SHA-256 lives in + a claim annotation. +- **Why:** no database, no separate account service; deleting the claim is account deletion and + cascades everything. +- **Cost:** no password reset, no admin API, no audit log of its own; Kubernetes API load grows + with users. Fine for a reference pattern, not for a product. + +### ADR-6: Gateway cookie session instead of bearer tokens in the browser + +- **Decision:** browsers get `dsh_gw_session=:`, `HttpOnly`, `SameSite=Lax` + (`Secure` when enabled); API clients may still use `Authorization: Bearer`. +- **Why:** tokens never appear in URLs or JS-readable storage, and `SameSite=Lax` plus a strict + `Origin`/`Referer` check on cookie-authenticated mutations blocks basic CSRF. +- **Cost:** cookie-based auth needs TLS and correct `COOKIE_SECURE`; there is no signed-cookie + layer, so the gateway trusts the token hash comparison for validation. + +## Security and RBAC + +The gateway runs with a namespace-scoped ServiceAccount (`50-gateway.yaml`): + +| API group | Resource | Verbs | Why | +| --- | --- | --- | --- | +| `extensions.agents.x-k8s.io` | `sandboxclaims` | `get`, `list`, `create`, `delete` | signup, state, account deletion | +| `agents.x-k8s.io` | `sandboxes` | `get`, `list`, `patch` | resolve DNS/pod, wake/suspend | +| `""` (core) | `pods` | `get`, `list` | resolve the pod behind a sandbox | +| `""` (core) | `pods/log` | `get` | launch-token handshake | + +Notes and limits: + +- **Pod logs are sensitive.** They contain the DSH launch token and possibly model output. The + permission is scoped to one namespace, but anyone who can read gateway memory or logs can + impersonate users. Prefer the upstream token API when available. +- **No cluster-wide access, no secret reads.** The gateway does not read `dsh-provider-keys`. +- **Browser-facing hardening:** `HttpOnly`/`SameSite=Lax`/optional `Secure` cookies, + timing-safe SHA-256 token comparison, strict same-origin check on cookie-auth mutations, + request-body limit (`MAX_BODY_MB`), hop-by-hop header stripping, upstream `Set-Cookie` + dropped, DSH `Location` headers rewritten under `/u//`. +- **Host allow-list:** DSH rejects unknown `Host` headers. The gateway forwards the browser's + `Host`, so every public hostname must be in `DSH_TRUSTED_HOSTS`; `localhost`/`127.0.0.1` are + always trusted. +- **TLS is external.** The gateway serves plain HTTP; terminate TLS at the Ingress and set + `COOKIE_SECURE=true`. Without TLS, session cookies are replayable on the network. +- **Tenant isolation** is pod + PVC separation; all tenants share one namespace, one DSH binary, + and one `DEEPSEEK_API_KEY`. Add NetworkPolicies and per-tenant credentials before calling it + multi-tenant-grade. +- **No gateway rate limiting.** Signup can create claims until the cluster or warm pool is + exhausted; put limits at the ingress for real deployments. + +## Failure modes + +| Failure | Symptom | Behavior / mitigation | +| --- | --- | --- | +| DSH log line changed or rotated away | Proxy returns `503` (missing launch token) | Fix the parser or restart the pod so the token line is re-emitted | +| Probe on `/` | Pod `CrashLoopBackOff`, `401` in probe logs | Use `/favicon.svg` for readiness/liveness (shipped templates do) | +| Warm pool exhausted | JSON signup returns `202` (state `Waking`); form signup is slow | Increase `replicas` in the warm pool | +| Gateway restart | Idle sandboxes stop suspending; handshake cache lost | Rebuilt lazily on next request; consider a shared store for production | +| DSH auth cookie expired mid-session | Proxy `401` once, then recovers | Gateway refreshes the handshake once | +| Upstream pod name changed after resume | Cached cookie keyed by old pod UID is useless | Cache is invalidated on suspend; next request re-handshakes | +| Two gateway replicas | Double suspend/wake races | Keep `replicas: 1` until state is externalized | + +## Upstream feature request: remove log scraping + +DSH should expose the launch token as a first-class, machine-readable interface so platforms do +not need `pods/log` access. Proposed shapes (any one suffices): + +1. **Token file:** `dsh web --token-file /run/dsh/token` writes the current launch token (mode + `0600`) and rewrites it on rotation. +2. **Environment input:** accept `DSH_LAUNCH_TOKEN` (platform-generated) so the token never + has to be discovered. +3. **Loopback/trusted-host endpoint:** `GET /api/auth/launch-token` that only answers on the + pod's loopback or a trusted host and returns the token plus expiry. +4. **Token in a well-known file** under `DSH_HOME` with the same semantics as (1). + +Why this matters: log scraping is brittle (rotation, restart, format changes), requires a broad +`pods/log` permission that also exposes application output, and can leak tokens into log +pipelines. Until an upstream API exists, the gateway's parser, `DSH_LOG_TAIL_LINES`, and the +single-retry refresh are the supported workaround. + +## Evolution path + +1. Replace log scraping with the upstream token API (above), deleting the `pods/log` permission. +2. Move idle clocks, in-flight tracking, and handshake cache to a shared store; then scale the + gateway horizontally (sticky routing not required if state is shared). +3. Add per-tenant API keys or an egress proxy so a single key is not shared by all sandboxes. +4. Add NetworkPolicies (gateway → sandboxes only), TLS everywhere, rate limits, and audit + logging at the ingress. +5. If DSH grows absolute URLs or per-user cookies that cannot be rewritten, switch to + per-user subdomains with wildcard DNS/TLS. diff --git a/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml index dcd3fe0..dadc541 100644 --- a/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml +++ b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml @@ -50,20 +50,39 @@ spec: # gateway base URL if you want hard-coded absolute links instead. - name: DSH_PUBLIC_URL value: "" + # DSH Host/Origin-fences /api. The gateway forwards the browser's + # Host, so DSH must trust the public gateway host. Empty is right + # for port-forward/localhost; for the optional Ingress set this to + # the same host as 60-ingress.yaml (e.g. dsh.example.com). + - name: DSH_TRUSTED_HOSTS + value: "" + # Probes target /favicon.svg: DSH gates / behind its launch-token + # cookie and answers 401 to an unauthenticated GET, which would + # otherwise fail readiness and then get the pod killed by liveness. + # /favicon.svg is served without auth and is the cheapest liveness + # signal that proves the HTTP server is actually serving requests. readinessProbe: httpGet: - path: / + path: /favicon.svg port: 3080 initialDelaySeconds: 5 periodSeconds: 5 failureThreshold: 30 livenessProbe: httpGet: - path: / + path: /favicon.svg port: 3080 initialDelaySeconds: 15 periodSeconds: 15 failureThreshold: 3 + # The container only needs to serve HTTP; drop every capability and + # disallow privilege escalation. DSH writes only to DSH_HOME (PVC). + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault resources: requests: {cpu: 250m, memory: 512Mi} limits: {cpu: "1", memory: 1Gi} diff --git a/examples/dsh-agents-as-a-service/50-gateway.yaml b/examples/dsh-agents-as-a-service/50-gateway.yaml index 8ca6bb5..ffcf10f 100644 --- a/examples/dsh-agents-as-a-service/50-gateway.yaml +++ b/examples/dsh-agents-as-a-service/50-gateway.yaml @@ -1,15 +1,28 @@ # The gateway: control+data plane that turns the resources into a usable -# service. Build the image first (see README step 8): +# browser-facing service. It is a Starlette ASGI app (see gateway/) listening +# on :8080 and exposing: +# GET /healthz unauthenticated health probe +# GET / landing page (signup / token login) +# POST /users, POST /login, ... session cookie auth +# /u//... HTTP + WebSocket proxy to the user's DSH +# +# Image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway +# For local kind development you can build and load a local tag instead, then +# change `image` below to it: # docker build -t dsh-aaas-gateway:demo gateway/ -# kind load docker-image dsh-aaas-gateway:demo # (kind clusters) +# kind load docker-image dsh-aaas-gateway:demo apiVersion: v1 kind: ServiceAccount metadata: name: dsh-gateway namespace: dsh-demo --- -# Least privilege: claims are created/deleted per user; sandboxes are only -# read and patched (operatingMode flips for suspend/resume). +# Least privilege: +# - claims are created/deleted per user (signup/delete); +# - sandboxes are read, and patched only to flip operatingMode (suspend/resume); +# - pods are read, and pods/log is read to capture DSH's launch token from +# the startup log line (`dsh web: ...?token=...`). +# configmaps are not needed: all state is in memory / claim annotations. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -22,6 +35,12 @@ rules: - apiGroups: ["agents.x-k8s.io"] resources: ["sandboxes"] verbs: ["get", "list", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -43,7 +62,11 @@ metadata: name: dsh-gateway namespace: dsh-demo spec: - replicas: 1 # in-memory idle clock => single replica by design + # Single replica BY DESIGN: the idle clock, in-flight counters and the + # per-(user, pod, Host) DSH cookie cache are in-process memory. Running two + # replicas would fragment that state (double suspend/resume races, repeated + # log-token handshakes). Scale only after moving state to a shared store. + replicas: 1 selector: matchLabels: app: dsh-gateway @@ -61,7 +84,7 @@ spec: type: RuntimeDefault containers: - name: gateway - image: dsh-aaas-gateway:demo + image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: false @@ -69,17 +92,60 @@ spec: capabilities: drop: ["ALL"] ports: - - containerPort: 8080 + - name: http + containerPort: 8080 env: - name: NAMESPACE value: dsh-demo - name: POOL value: dsh-pool + # Seconds a sandbox may sit idle before the sweeper suspends it + # (pod deleted, PVC kept), and the max time a proxy request waits + # for a suspended sandbox to become Ready again. - name: IDLE_TIMEOUT value: "60" + - name: WAKE_TIMEOUT + value: "120" + # Container name inside the per-user sandbox pods whose logs carry + # the DSH launch token. + - name: DSH_CONTAINER + value: dsh-web + # Cookie dsh_gw_session is HttpOnly + SameSite=Lax. Set to "true" + # when this Service is reached over HTTPS (e.g. the optional + # 60-ingress.yaml with TLS); "false" keeps the plain-HTTP demo + # usable via port-forward. + - name: COOKIE_SECURE + value: "false" + # Reject proxied request bodies larger than this (413). + - name: MAX_BODY_MB + value: "32" + # /healthz is unauthenticated and returns 200 without touching the + # cluster, so it is safe for both readiness and liveness. + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 resources: requests: {cpu: 50m, memory: 128Mi} limits: {cpu: 500m, memory: 256Mi} + volumeMounts: + # readOnlyRootFilesystem leaves no writable HOME/tmp; the image sets + # HOME=/tmp, so back it with scratch space. No state lives here. + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/examples/dsh-agents-as-a-service/60-ingress.yaml b/examples/dsh-agents-as-a-service/60-ingress.yaml new file mode 100644 index 0000000..c7300db --- /dev/null +++ b/examples/dsh-agents-as-a-service/60-ingress.yaml @@ -0,0 +1,46 @@ +# OPTIONAL browser entry point: expose the gateway through Traefik. +# +# Requires 50-gateway.yaml (Service dsh-gateway:8080) and a Traefik install +# with an IngressClass named "traefik" (k3s ships one). Replace the placeholder +# host below with a name that resolves to your ingress address, and either +# supply a real TLS Secret or delete the `tls:` block for plain HTTP. +# +# Two things must match this host: +# 1. SandboxTemplate (10-sandboxtemplate.yaml): set DSH_TRUSTED_HOSTS to +# this host so DSH accepts the forwarded Host/Origin on /api. +# 2. Gateway Deployment (50-gateway.yaml): set COOKIE_SECURE to "true" +# once this Ingress serves HTTPS, so the session cookie is Secure. +# +# WebSocket support is REQUIRED: the browser talks to /u//... and DSH +# uses WebSockets for live session traffic. Traefik upgrades WS/SSE +# transparently for ordinary HTTP routers, but do NOT attach buffering +# middlewares (e.g. compress, buffering) to this router — they break streaming +# and upgrades. `path: /` Prefix covers the landing page and every per-user +# proxy path, so no per-sandbox Ingress is needed. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dsh-gateway + namespace: dsh-demo + # Typical Traefik + cert-manager wiring (uncomment and adapt): + # annotations: + # traefik.ingress.kubernetes.io/router.entrypoints: websecure + # cert-manager.io/cluster-issuer: letsencrypt-prod + annotations: {} +spec: + ingressClassName: traefik + tls: + - hosts: + - dsh.example.com # placeholder + secretName: dsh-example-tls # placeholder; create/PKI-provision this Secret + rules: + - host: dsh.example.com # placeholder + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: dsh-gateway + port: + number: 8080 diff --git a/examples/dsh-agents-as-a-service/README.md b/examples/dsh-agents-as-a-service/README.md index 21c0072..a7f47c0 100644 --- a/examples/dsh-agents-as-a-service/README.md +++ b/examples/dsh-agents-as-a-service/README.md @@ -11,10 +11,11 @@ sessions, config) surviving on a per-user PVC at `/home/node/.dsh`. It ships in two layers: 1. **The resource layer** (steps 1–7): pure manifests + kubectl, so every mechanism is visible. -2. **The service layer** (step 8): a small included **gateway** (`gateway/`) that makes it - directly usable — signup API, per-user bearer tokens, a `/u/{user}/**` proxy that - transparently **wakes suspended instances on connect**, and an idle sweeper that suspends - them again. +2. **The service layer** (step 8): a small included **gateway** (`gateway/`) that makes the + platform browser-usable — a signup page/API, `HttpOnly` cookie sessions, a `/u/{user}/**` + proxy for HTTP **and WebSocket** that transparently **wakes suspended instances on + connect**, and an idle sweeper that suspends them again. The gateway performs the DSH + launch-token handshake server-side, so the browser never sees the token. What it demonstrates, feature by feature: @@ -37,11 +38,14 @@ What it demonstrates, feature by feature: instead of splitting `/v1/*` (API) from the dashboard. - **One shared secret**: only `DEEPSEEK_API_KEY` is injected (via `envFrom`); there is no dashboard basic-auth, session-cookie secret, or API-server key. -- **Host forwarding**: the gateway forwards the original `Host` + `X-Forwarded-*` (and keeps - `DSH_PUBLIC_URL` empty) so DSH generates reachable links from the request. -- **WebSocket caveat**: the requests-based streaming proxy handles SSE but not WebSocket - upgrades. For full DSH web-GUI interactivity, front the gateway with a path-based Ingress - per user (or a WS-capable proxy). +- **Launch-token handshake**: DSH gates `/` behind a token it prints only in its logs. The + gateway reads it, performs the handshake, and keeps the resulting auth cookie server-side; + the browser authenticates to the gateway with its own cookie. See + [How the DSH handshake works](#how-the-dsh-handshake-works). +- **Host allow-list**: DSH rejects unknown `Host` headers. The gateway forwards the browser's + `Host`, so the public hostname must be added to `DSH_TRUSTED_HOSTS` in the template. +- **Probes target `/favicon.svg`**: `/` returns `401` without the launch token, so a probe on + `/` crash-loops the pod. `/favicon.svg` is unauthenticated. ## Prerequisites @@ -55,7 +59,7 @@ What it demonstrates, feature by feature: ``` Or run everything below in one shot with `./run-test-kind.sh` (expects a kind cluster + the -manifest above already applied). +manifest above already applied; it creates and deletes the namespace itself). ## 1. Install the platform resources @@ -111,14 +115,20 @@ Three things worth noticing: ## 3. Talk to the instance +DSH Web needs its launch token on first contact — a bare `/` returns `401`. The token is in the +pod logs: + ```bash -kubectl -n dsh-demo port-forward "$SB" 18081:3080 -# open http://localhost:18081 — DSH Web for alice +kubectl -n dsh-demo logs "$SB" | grep -m1 '?token=' +# dsh web: http://:3080/?token=... +kubectl -n dsh-demo port-forward "pod/$SB" 18081:3080 +# open http://localhost:18081/?token=... (from the log line) ``` > **Local port note:** DSH Web listens on `3080` **in-cluster**. If `3080` is already in use > on your workstation (e.g. by a local DSH), forward to a different local port (here `18081`), -> never `3080`. +> never `3080`. `localhost` is always in DSH's trusted-host list, so the forwarded request is +> accepted. ## 4. Write state on the PVC @@ -153,38 +163,169 @@ kubectl -n dsh-demo get sandboxclaim dsh-mallory \ kubectl delete -f 40-claim-rejected.yaml ``` -## 8. The gateway: signup, proxy, idle-suspend, wake-on-connect +## 8. The gateway: signup, browser proxy, idle-suspend, wake-on-connect -Build the gateway image and deploy it: +### Deploy it ```bash -docker build -t dsh-aaas-gateway:demo gateway/ -kind load docker-image dsh-aaas-gateway:demo # (kind clusters) +# Published image (on kind, load it into the cluster; on a real cluster it is +# pulled from GHCR): +docker pull ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway +kind load docker-image ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway --name + +# ... or build it locally and tag it exactly as 50-gateway.yaml expects: +docker build -t ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway gateway/ +kind load docker-image ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway --name + kubectl apply -f 50-gateway.yaml kubectl -n dsh-demo rollout status deploy/dsh-gateway --timeout=120s kubectl -n dsh-demo port-forward svc/dsh-gateway 18080:8080 ``` -Then: +> The gateway binds `8080`; the local port-forward uses `18080` and never `3080`. +> +> **Applying over an existing install?** Run `kubectl diff -f 10-sandboxtemplate.yaml` +> first. Cluster-specific fields such as `nodeAffinity` and the warm-pool replica count are +> intentionally not in the repo manifests; preserve them with a targeted patch instead of a +> blind apply. Moving the probes to `/favicon.svg` is the crash-loop fix. + +### Browser flow + +Open . The landing page has a one-field signup form. Enter `bob`, and +the gateway: + +1. creates the `SandboxClaim` over the warm pool, +2. waits for it to become `Ready`, +3. sets an `HttpOnly`, `SameSite=Lax` cookie `dsh_gw_session=:` on the browser, +4. redirects to `/u/bob/`, where the DSH Web UI loads through the gateway. + +From then on every request under `/u/bob/` (assets, API calls, SSE, WebSocket) is proxied to +bob's sandbox. The DSH launch token and DSH's own auth cookie stay in the gateway; the browser +only holds the gateway cookie. + +The same flow with `curl`, no JavaScript (the session cookie is the only credential): ```bash -# signup → claim over the warm pool + one-time bearer token -SIGNUP=$(curl -s -X POST http://localhost:18080/users -H 'Content-Type: application/json' -d '{"user":"bob"}') -TOKEN=$(echo "$SIGNUP" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p') - -# proxy to bob's DSH Web (the gateway wakes it if suspended) -curl -s -H "Authorization: Bearer $TOKEN" http://localhost:18080/u/bob/ - -# tokenless is rejected +JAR=$(mktemp) +# signup (form POST, exactly what the landing page sends) +curl -s -i -c "$JAR" -H 'Origin: http://localhost:18080' \ + -d 'user=bob' http://localhost:18080/users +# proxy: bob's DSH Web through the gateway +curl -s -b "$JAR" http://localhost:18080/u/bob/ | head -5 +# without the cookie the gateway fails closed curl -s -o /dev/null -w '%{http_code}\n' http://localhost:18080/u/bob/ # 401 - -# after IDLE_TIMEOUT (60s) the sweeper suspends bob -curl -s -H "Authorization: Bearer $TOKEN" http://localhost:18080/users/bob # state: Suspended - +# derived state: Ready / Waking / Suspended / ... +curl -s -b "$JAR" http://localhost:18080/users/bob # delete bob (cascades sandbox + PVC) -curl -s -X DELETE -H "Authorization: Bearer $TOKEN" http://localhost:18080/users/bob +curl -s -b "$JAR" -H 'Origin: http://localhost:18080' -X DELETE http://localhost:18080/users/bob ``` +API clients that prefer bearer tokens can use `POST /users` with a JSON body +(`{"user":"bob"}`) and the returned one-time token, `POST /login` to exchange a token for a +cookie, and `Authorization: Bearer ` on subsequent calls. + +### Gateway API + +| Route | Purpose | +| --- | --- | +| `GET /healthz` | Liveness/readiness; no cluster access | +| `GET /` | Landing page: signup, token login, logout | +| `POST /users` | Signup. JSON or form; `201` (JSON), `202` (waking), `303` (browser redirect), `409` if user exists; sets the session cookie | +| `POST /login` | Exchange a bearer token for the session cookie | +| `POST /logout` | Clear the session cookie | +| `GET /users/{user}` | Derived state (`Ready`/`Waking`/`Suspended`) | +| `DELETE /users/{user}` | Delete claim (cascades sandbox + PVC) | +| `GET /u/{user}` | `308` redirect to `/u/{user}/` | +| `GET /u/{user}/**` | HTTP/SSE proxy to the user's DSH Web | +| `WS /u/{user}/**` | WebSocket proxy to the user's DSH Web | + +### How the DSH handshake works + +DSH does not expose an API for its launch token; it prints a URL containing `?token=…` in the +pod logs. For each `(user, pod UID, Host)` the gateway: + +1. resolves the claim → sandbox → pod (annotation `agents.x-k8s.io/pod-name`, or the sandbox + name as pod name), +2. reads the pod's logs (last `DSH_LOG_TAIL_LINES` lines) and parses the most recent + `dsh web: …?token=…` line, +3. requests `http://:3080/?token=…` with the browser's `Host`, +4. captures DSH's `dsh-auth-*` `Set-Cookie` response and caches it, +5. injects that cookie (and strips the browser's `Cookie`/`Authorization` and any `?token=`) + on every proxied HTTP and WebSocket request, +6. refreshes the handshake once if DSH answers `401`, and drops the cached cookie when the + sandbox suspends or is deleted. + +This is why the gateway's RBAC includes `pods` `get`/`list` and `pods/log` `get`. It is also +the most fragile part of the design; see [Limitations](#limitations). + +### DSH_TRUSTED_HOSTS + +DSH rejects requests whose `Host` is not allow-listed (`localhost` and `127.0.0.1` are always +trusted). Because the gateway forwards the browser's `Host`, set the public hostname in the +template: + +```yaml +env: + - name: DSH_TRUSTED_HOSTS + value: "dsh.example.com" # comma or space separated; empty for port-forward only +``` + +The same applies to a local `docker run`: the printed token URL uses `DSH_PUBLIC_URL` if set, +otherwise `http://localhost:${DSH_PORT}`. + +### WebSocket + +The gateway is a Starlette ASGI app and proxies WebSocket upgrades with a bidirectional pump, so +the DSH web GUI is fully interactive. Requirements: + +- the ingress/load balancer in front of the gateway must pass `Upgrade`/`Connection` headers; + `60-ingress.yaml` is a starting point (Traefik handles upgrades by default), +- idle suspension is disabled for users with in-flight proxied requests or open WebSockets, so + a long-running chat stream is not cut off mid-turn, +- missing/invalid gateway auth is rejected right after the upgrade with close code `4401` + (WebSocket auth has no HTTP `401` path); cross-origin or Origin-less upgrades with the cookie + are rejected the same way (CSWSH protection), so clients must send a matching `Origin`; + upstream bridging failures close with `1011`. + +### Path routing + +Each user's DSH Web is served under `/u//`. DSH injects `` and uses +relative asset/API URLs, so the UI works under a path prefix without a per-user hostname or +wildcard DNS. The gateway also rewrites absolute-path `Location` headers into `/u//…` and +redirects `/u/` to the trailing-slash form with `308` so relative URLs resolve correctly. + +## Troubleshooting + +| Symptom | Cause / fix | +| --- | --- | +| Pod `CrashLoopBackOff`, probes failing with `401` | Probe path is `/`; change readiness/liveness to `/favicon.svg` | +| Browser shows `401` from DSH through the gateway | Gateway handshake failed — check `kubectl -n dsh-demo logs deploy/dsh-gateway` and that RBAC grants `pods/log` | +| `502` / `upstream unavailable` | Sandbox pod is not Ready (wake timed out) or the Service DNS is stale; check `kubectl get sandbox`, `WAKE_TIMEOUT` | +| `503` with `Retry-After` | The launch token was not found in the pod logs yet; the gateway retries after a short delay. Check the pod logs and `DSH_LOG_TAIL_LINES` | +| `409` on signup | User already has a claim; use `POST /login` with the existing token or delete the user first | +| Cookie set but browser keeps getting `401` | `COOKIE_SECURE=true` while serving over plain HTTP (secure cookies are not sent); use HTTPS or set `false` for local demos | +| WebSocket closes immediately (`4401`) | Missing/expired gateway cookie, or the upgrade's `Origin` does not match `Host` (CSWSH check); DSH rejecting `Host` instead surfaces as close `1011` — add the host to `DSH_TRUSTED_HOSTS` | +| Signup is slow / `202` then `Waking` | Warm pool exhausted (`kubectl get sandboxes -l agents.x-k8s.io/warm-pool-sandbox`); scale the pool | +| `403` from the Kubernetes API in gateway logs | The gateway ServiceAccount is missing claim/sandbox/pod-log verbs (see `50-gateway.yaml`) | + +## Limitations + +Deliberately **not** production code: + +- **Single replica.** Idle clocks, in-flight tracking, and the handshake cache are in memory. + Running two gateway replicas would double-suspend or lose wake tracking; scale up only after + moving that state to a shared store. +- **Log scraping.** The launch token is recovered from pod logs. Log rotation, a restarted pod, + or a changed DSH log format can break the handshake until the next `401` refresh. +- **Shared provider key.** `DEEPSEEK_API_KEY` is shared by all tenants via the template. Real + multi-tenancy needs per-tenant credentials or an egress proxy. +- **No TLS in the gateway.** Terminate TLS at an Ingress/reverse proxy and set + `COOKIE_SECURE=true`. +- **In-memory user directory.** Users are claims + annotations; there is no database, password + reset, or admin API. Tokens are stored only as SHA-256 hashes in claim annotations. +- **Bearer tokens in URLs.** Avoid putting the signup token in a URL/log; the browser flow uses + the `HttpOnly` cookie instead. + ## Cost model The same economics as the upstream pattern: suspend-when-idle plus warm pools turn an diff --git a/examples/dsh-agents-as-a-service/gateway/Dockerfile b/examples/dsh-agents-as-a-service/gateway/Dockerfile index f83f0bf..6a6fd2d 100644 --- a/examples/dsh-agents-as-a-service/gateway/Dockerfile +++ b/examples/dsh-agents-as-a-service/gateway/Dockerfile @@ -1,7 +1,18 @@ FROM python:3.12-slim + +# Read-only-rootfs friendly: no .pyc writes, no home directory writes. +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + HOME=/tmp + WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY gateway.py . + +EXPOSE 8080 +# Run as a fixed non-root UID; HOME=/tmp keeps caches out of the image. USER 10000 -CMD ["python", "gateway.py"] + +CMD ["uvicorn", "gateway:create_app", "--factory", \ + "--host", "0.0.0.0", "--port", "8080", "--no-access-log"] diff --git a/examples/dsh-agents-as-a-service/gateway/README.md b/examples/dsh-agents-as-a-service/gateway/README.md new file mode 100644 index 0000000..a1fe4ab --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/README.md @@ -0,0 +1,184 @@ +# DSH agents-as-a-service gateway + +A single-file, browser-usable control + data plane for +[DeepSeek Harness Web (DSH)](https://github.com/deepseek-ai/deepseek-harness) +sandboxes running behind the `agent-sandbox` CRDs. + +It does three things: + +1. **Users** — creates a `SandboxClaim` from a warm pool, mints a bearer token + (only its SHA-256 is stored as a claim annotation), reports derived state, + and deletes the claim (cascading the sandbox and PVC). +2. **Browser sessions** — exchanges a user token for an HttpOnly session + cookie so the DSH UI works in a browser at `/u//`. Automation can + still use `Authorization: Bearer `. +3. **Reverse proxy** — proxies HTTP (including SSE) and WebSocket traffic to + the sandbox's DSH Web server on `:3080`, including DSH's own cookie + handshake. + +DSH is pinned to `dsh-v0.2.1-rc.2` in the example manifests. It binds `:3080`, +serves `/`, gates `/api` behind a `dsh-auth-*` cookie and requires a WebSocket +at `/api/remote.mux`. + +## Flow + +``` +browser / curl gateway cluster + | | | + | POST /users {"user":"alice"} | | + |------------------------------>| create SandboxClaim ---------->| + | | wait Ready | + | 201 {token} + session cookie | | + |<------------------------------| | + | GET /u/alice/ (cookie) | | + |------------------------------>| claim -> sandbox -> pod | + | | read pod logs (pods/log) | + | | find "dsh web: ...?token=..." | + | | GET http://:3080/?token= | + | | capture dsh-auth-* Set-Cookie | + | | proxy with Host/Origin kept | + | HTML / SSE / WebSocket |<----------------------------->| + |<------------------------------| | +``` + +The launch token is printed by DSH to its pod stdout: + +``` +dsh web: http://:3080/?token= +``` + +The gateway reads the tail of the pod logs, parses the **last** such line and +performs the `?token=` handshake itself with the browser's `Host` header. The +resulting `dsh-auth-*` cookie is cached per `(user, pod UID, Host)` and injected +into proxied requests. The browser never sees the launch token or the bearer +token. A failed handshake returns `503` with `Retry-After: 10`. + +## API + +| Method | Path | Auth | Purpose | +| --- | --- | --- | --- | +| `GET` | `/healthz` | none | Liveness/readiness. Never touches the cluster. | +| `GET` | `/` | none | Landing page with signup + token login forms. | +| `POST` | `/users` | none | Signup. JSON body `{"user": ""}` or form field `user`. JSON returns `201` when Ready, `202` when accepted but still Waking; form returns `303` to `/u//`. Both set the session cookie. `409` if the user exists, `400` if invalid, `413` if the body is too large. | +| `POST` | `/login` | none | Form (`user`, `token`) returns `303` to `/u//`; JSON returns `200`. Sets the session cookie. `401` on a bad token, `404` for an unknown user. | +| `POST` | `/logout` | none | Clears the session cookie and `303`s to `/`. | +| `GET` | `/users/{user}` | bearer or cookie | Derived state: `Provisioning`, `Waking`, `Ready`, `Suspending`, `Suspended`. | +| `DELETE` | `/users/{user}` | bearer or cookie | Delete the claim (sandbox + PVC cascade). Cookie-auth mutations require a same-origin `Origin`/`Referer`. | +| `GET` | `/u/{user}` | — | `308` redirect to `/u/{user}/`. | +| `*` | `/u/{user}/{path}` | bearer or cookie | HTTP proxy to DSH. Waits for a suspended sandbox to wake. Wakes on connect. | +| `WS` | `/u/{user}/{path}` | bearer or cookie | WebSocket proxy (e.g. `/api/remote.mux`). | + +### Auth + +* `Authorization: Bearer ` is checked with a timing-safe SHA-256 + comparison against the `aaas.example.com/token-sha256` claim annotation. +* The browser session cookie is + `dsh_gw_session=:` (HttpOnly, SameSite=Lax, Path=/, optional + Secure). A session for user A never authorizes user B. +* Cookie-authenticated **mutations** (`POST`, `PUT`, `PATCH`, `DELETE`) and + cookie-authenticated **WebSocket upgrades** require the request `Origin` + (or `Referer` fallback) to match the `Host` header. Missing or mismatched + cross-origin metadata is rejected (`403` HTTP, close code `4401` for WS). +* The gateway's bearer token, the browser `Cookie` header and any `?token=` + query parameter are never forwarded upstream. Only the cached `dsh-auth-*` + cookie is injected. +* Upstream `Set-Cookie` headers are dropped. `Location` headers that are + absolute paths are rewritten under `/u//`, and any `token` query + parameter is stripped. + +### Proxy failure semantics + +| Situation | Result | +| --- | --- | +| Unknown user / claim | `404` | +| Missing or invalid credentials | `401` | +| Cross-origin cookie mutation | `403` | +| Request body over `MAX_BODY_MB` | `413` | +| Sandbox not provisioned, wake timeout, handshake failure | `503` + `Retry-After: 10` | +| Upstream connection/protocol failure | `502` | +| WS auth failure (unknown user, bad token, cross-origin) | accept, then close `4401` | +| WS upstream/handshake failure | accept, then close `1011` | + +A close code can only be delivered after the WebSocket upgrade is accepted; +Starlette would turn an early close into an HTTP 403, which is why WS auth +failures are `101`, then close `4401`. + +## Configuration + +All values are read from the environment by `Settings.from_env()`; every one +can also be injected into `create_app` for tests. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `NAMESPACE` | `dsh-demo` | Namespace holding the claims, sandboxes and pods. | +| `POOL` | `dsh-pool` | `warmPoolRef.name` for new claims. | +| `IDLE_TIMEOUT` | `60` | Seconds of inactivity before the sweeper suspends a sandbox. | +| `WAKE_TIMEOUT` | `120` | Seconds to wait for a sandbox to become Ready (signup and wake-on-connect). | +| `DSH_PORT` | `3080` | Upstream DSH port. | +| `DSH_CONTAINER` | `dsh-web` | Container whose logs carry the launch token. | +| `DSH_LOG_TAIL_LINES` | `2000` | How many log lines to scan for the launch token. | +| `COOKIE_NAME` | `dsh_gw_session` | Browser session cookie name. | +| `COOKIE_SECURE` | `false` | Add `Secure` to the session cookie when served over TLS. | +| `COOKIE_MAX_AGE` | `604800` | Session cookie lifetime in seconds. | +| `MAX_BODY_MB` | `32` | Maximum buffered request body (and `413` threshold). | +| `HTTP_TIMEOUT` | `300` | Upstream connect/read timeout in seconds. | +| `SWEEP_INTERVAL` | `15` | Seconds between idle sweeps. | + +### Required RBAC + +The gateway's ServiceAccount needs, in its namespace: + +* `sandboxclaims` (`extensions.agents.x-k8s.io`): get, list, create, delete +* `sandboxes` (`agents.x-k8s.io`): get, patch +* `pods`: get +* `pods/log`: get + +## Running locally + +The module has no import-time cluster access and the Kubernetes adapter is +lazy, so `/healthz` works without a kubeconfig: + +```bash +python -m pip install -r requirements-dev.txt +python -m pytest -q +python -m ruff check . + +# Uvicorn factory (loads in-cluster credentials only on the first claim call) +uvicorn gateway:create_app --factory --host 0.0.0.0 --port 8080 +``` + +Container image: + +```bash +docker build -t dsh-gateway examples/dsh-agents-as-a-service/gateway +``` + +The image runs as UID `10000`, sets `PYTHONDONTWRITEBYTECODE=1` and +`HOME=/tmp` (read-only-rootfs compatible), listens on `8080`, and disables +uvicorn's access log so bearer tokens in URLs are not logged. + +## Known limitations + +* **Log scraping is fragile.** The launch token is discovered from pod logs. + Log rotation, a very chatty container, or DSH changing its startup message + breaks the handshake. Prefer a first-class token/credential API when DSH + offers one. +* **Single replica, in-memory state.** The handshake cookie cache, idle + activity clocks and in-flight counters live in one process. Running more + than one replica would need shared state (or sticky routing), otherwise each + replica handshakes independently and the sweeper may race another replica. +* **In-memory token cache has no TTL.** It is keyed by pod UID, so pod + restarts invalidate it naturally; otherwise entries survive until the pod or + claim is deleted. +* **No TLS termination.** Run behind an ingress/load balancer; set + `COOKIE_SECURE=true` there. +* **Origin-based CSRF protection only.** Cookie-auth mutations require a + same-origin `Origin`/`Referer`; there is no per-form CSRF token, rate + limiting or login throttling. +* **HTTP request bodies are buffered** up to `MAX_BODY_MB`; uploads are not + streamed. Responses (including SSE) are streamed. +* **Upstream cookies are not forwarded.** DSH cannot rotate its own + `dsh-auth-*` cookie through the gateway; on an upstream `401` the gateway + re-runs the handshake once and replays the request. +* **Only DSH on `:3080` is proxied.** There is no dashboard/API-port split and + no platform API key injection. diff --git a/examples/dsh-agents-as-a-service/gateway/gateway.py b/examples/dsh-agents-as-a-service/gateway/gateway.py index b52a14f..ced9f86 100644 --- a/examples/dsh-agents-as-a-service/gateway/gateway.py +++ b/examples/dsh-agents-as-a-service/gateway/gateway.py @@ -1,348 +1,1272 @@ -# Minimal agents-as-a-service gateway for DeepSeek Harness Web (DSH). -# -# A compact, single-file control+data plane adapted from the upstream -# hermes-agents-as-a-service gateway (kubernetes-sigs/agent-sandbox). It turns -# the DSH sandbox resources into a usable multi-user service: -# -# POST /users {"user": "alice"} signup: create a SandboxClaim over the -# warm pool, wait Ready, mint a bearer -# token (only its SHA-256 is stored, as a -# claim annotation) -# GET /users/ derived state: Ready/Waking/Suspended/... -# (requires the user's bearer token) -# DELETE /users/ delete the claim (cascades sandbox + PVC; -# requires the user's bearer token) -# ANY /u// per-user proxy. Requests to a suspended -# sandbox are HELD while it resumes -# (wake-on-connect); everything is proxied -# to the DSH Web UI on :3080. -# -# A background sweeper suspends sandboxes idle for IDLE_TIMEOUT seconds — -# suspend deletes only the pod; the PVC (and the user's state) survives. -# -# Differences from the Hermes original: -# - Single upstream on :3080 (no OpenAI-compatible :8642 / dashboard :9119 -# split, and no platform API key to inject). -# - The original Host header and X-Forwarded-* are forwarded (the gateway's -# own bearer token and any ?token= are still stripped) so DSH generates -# reachable links from the request Host. -# - Requests streaming proxies SSE but does NOT handle WebSocket upgrades; -# a per-user path-based Ingress is the alternative for full WS support. -# -# Deliberately NOT production code: single replica, in-memory idle clock, -# no TLS. +"""Browser-usable gateway for DeepSeek Harness Web (DSH) sandboxes. +This is the control + data plane in front of the agent-sandbox CRDs. It is a +single teaching-friendly module with three jobs: + +* **Users** -- create a ``SandboxClaim`` over a warm pool, mint a bearer token + (only its SHA-256 is stored, as a claim annotation), report derived state, + and delete the claim (which cascades the sandbox and its PVC). +* **Browser sessions** -- exchange a user token for an HttpOnly session cookie + so the DSH UI can be opened in a browser at ``/u//``. Browser requests + authenticate with that cookie; automation can keep using + ``Authorization: Bearer ``. +* **Reverse proxy** -- HTTP (including SSE) and WebSocket proxying to the + sandbox's DSH Web server on ``:3080``. DSH itself is cookie-gated: the + gateway scrapes the launch token from the pod logs, performs the ``?token=`` + handshake server-side, caches the resulting ``dsh-auth-*`` cookie, and + injects it upstream. The browser never sees the launch or bearer token. + +A background sweeper suspends sandboxes idle for ``IDLE_TIMEOUT`` seconds. +Suspending deletes the pod only; the PVC (and the user's state) survives. + +Deliberately NOT production code: single replica, in-memory state and token +cache, no TLS termination, and launch-token discovery by log scraping. + +Run locally with:: + + uvicorn gateway:create_app --factory --host 0.0.0.0 --port 8080 + +The factory signature (``Settings``, ``Kubernetes``, an httpx client and the +websockets connector are all injectable) is what makes the test suite +cluster-free. +""" + +from __future__ import annotations + +import asyncio +import contextlib import hashlib +import html +import json import os import re import secrets -import threading import time +from dataclasses import dataclass, field +from urllib.parse import parse_qs, parse_qsl, quote, urlencode, urlsplit, urlunsplit -import requests -from flask import Flask, Response, jsonify, request -from kubernetes import client, config +import anyio +import httpx +import websockets +from kubernetes import client as k8s_client +from kubernetes import config as k8s_config +from kubernetes.client.exceptions import ApiException +from starlette.applications import Starlette +from starlette.datastructures import Headers +from starlette.requests import Request +from starlette.responses import ( + HTMLResponse, + JSONResponse, + RedirectResponse, + Response, + StreamingResponse, +) +from starlette.routing import Route, WebSocketRoute +from starlette.websockets import WebSocket + +# --- Kubernetes / DSH constants ------------------------------------------- GROUP, VERSION = "extensions.agents.x-k8s.io", "v1beta1" CORE_GROUP = "agents.x-k8s.io" -NAMESPACE = os.environ.get("NAMESPACE", "dsh-demo") -POOL = os.environ.get("POOL", "dsh-pool") -IDLE_TIMEOUT = int(os.environ.get("IDLE_TIMEOUT", "60")) -WAKE_TIMEOUT = int(os.environ.get("WAKE_TIMEOUT", "120")) TOKEN_ANNOTATION = "aaas.example.com/token-sha256" -# RFC 1123 label: lowercase alphanumerics and '-', must start/end -# alphanumeric. The claim name is "dsh-", so cap the user part at -# 63 - len("dsh-") to keep the claim a valid Kubernetes object name. +POD_NAME_ANNOTATION = "agents.x-k8s.io/pod-name" +USER_LABEL = "sandbox.users.io/dsh-user" + +# RFC 1123 label: lowercase alphanumerics and "-", must start/end alphanumeric. +# The claim name is "dsh-", so cap the user part to keep the claim a +# valid Kubernetes object name. DNS1123_LABEL = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") MAX_USER_LEN = 63 - len("dsh-") -try: - config.load_incluster_config() -except config.ConfigException: - config.load_kube_config() -crd = client.CustomObjectsApi() +# Headers a proxy must not forward in either direction (RFC 7230 section 6.1). +HOP_BY_HOP = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) +SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) +# Headers the gateway owns: never forwarded from the browser, overwritten with +# the gateway's own view of the connection instead. +OWNED_REQUEST_HEADERS = HOP_BY_HOP | { + "authorization", + "cookie", + "host", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-port", + "x-forwarded-proto", + "x-real-ip", +} -app = Flask(__name__) -# Cap proxied request bodies: Flask rejects anything larger with 413 before -# request.get_data() would buffer it in memory. -app.config["MAX_CONTENT_LENGTH"] = \ - int(os.environ.get("MAX_BODY_MB", "32")) * 1024 * 1024 -last_activity: dict[str, float] = {} -# Admitted proxy requests per user. While > 0 the sweeper must not suspend: -# a request being served (or still streaming) after IDLE_TIMEOUT is -# activity, not idleness. The sweeper checks-and-suspends under the same -# lock that admission increments under, so the pair is atomic. -in_flight: dict[str, int] = {} -flight_lock = threading.Lock() +_DSH_WEB_LINE = re.compile(r"dsh web:\s+(?Phttps?://\S+)") -def end_flight(user: str): - with flight_lock: - in_flight[user] = max(in_flight.get(user, 1) - 1, 0) - last_activity[user] = time.time() +class UserExists(Exception): + """The claim already exists (a 409 from the API server).""" + + +class BodyTooLarge(Exception): + """The request body exceeded ``MAX_BODY_MB``.""" + + +class HandshakeError(Exception): + """DSH's launch token or auth cookie could not be obtained.""" + + +class SandboxUnavailable(Exception): + """The sandbox is not (yet) ready to serve traffic.""" + + +class UpstreamError(Exception): + """The upstream DSH connection failed.""" + + +# --- Pure helpers (unit-testable without an app) -------------------------- + + +def valid_user(user: str) -> bool: + """A user is usable as a claim-name suffix (dns-1123 label, length-capped).""" + return ( + isinstance(user, str) + and len(user) <= MAX_USER_LEN + and DNS1123_LABEL.fullmatch(user) is not None + ) def claim_name(user: str) -> str: return f"dsh-{user}" -def get_claim(user: str): - try: - return crd.get_namespaced_custom_object( - GROUP, VERSION, NAMESPACE, "sandboxclaims", claim_name(user)) - except client.ApiException as e: - if e.status == 404: - return None - raise +def claim_user(claim: dict) -> str: + return claim["metadata"]["name"].removeprefix("dsh-") -def get_sandbox(name: str): - return crd.get_namespaced_custom_object( - CORE_GROUP, VERSION, NAMESPACE, "sandboxes", name) - - -def sandbox_of(claim): - name = (claim.get("status") or {}).get("sandbox", {}).get("name") - if not name: - return None - try: - return get_sandbox(name) - except client.ApiException as e: - if e.status == 404: # stale claim status, e.g. mid-cascade-delete - return None - raise - - -def condition(obj, ctype: str) -> bool: +def condition(obj: dict, ctype: str) -> bool: for c in (obj.get("status") or {}).get("conditions", []): if c.get("type") == ctype: return c.get("status") == "True" return False -def derive_state(sandbox) -> str: +def derive_state(sandbox: dict | None) -> str: """The state machine a platform shows users (mode x conditions).""" if sandbox is None: return "Provisioning" - mode = sandbox["spec"].get("operatingMode", "Running") + mode = sandbox.get("spec", {}).get("operatingMode", "Running") if mode == "Suspended": return "Suspended" if condition(sandbox, "Suspended") else "Suspending" return "Ready" if condition(sandbox, "Ready") else "Waking" -def set_operating_mode(sandbox_name: str, mode: str): - crd.patch_namespaced_custom_object( - CORE_GROUP, VERSION, NAMESPACE, "sandboxes", sandbox_name, - {"spec": {"operatingMode": mode}}) - - -def wait_ready(user: str, timeout: int): - deadline = time.time() + timeout - while time.time() < deadline: - claim = get_claim(user) - sandbox = sandbox_of(claim) if claim else None - if sandbox is not None and derive_state(sandbox) == "Ready": - return sandbox - time.sleep(1) - return None - - -def authorized(claim) -> bool: - # Header-only on purpose: a ?token= query param would leak the bearer - # token into access logs, browser history and Referer headers. - token = request.headers.get("Authorization", "").removeprefix("Bearer ") - want = (claim["metadata"].get("annotations") or {}).get(TOKEN_ANNOTATION, "") +def token_matches(claim: dict, token: str | None) -> bool: + """Timing-safe check of a user token against the stored SHA-256.""" + want = (claim.get("metadata", {}).get("annotations") or {}).get( + TOKEN_ANNOTATION, "" + ) + if not token or not want: + return False got = hashlib.sha256(token.encode()).hexdigest() - return bool(token) and secrets.compare_digest(got, want) + return secrets.compare_digest(got, want) -@app.post("/users") -def create_user(): - user = (request.get_json(silent=True) or {}).get("user", "") - if not isinstance(user, str) or not DNS1123_LABEL.fullmatch(user) \ - or len(user) > MAX_USER_LEN: - return jsonify(error="body must be {'user': ''}"), 400 - if get_claim(user) is not None: - return jsonify(error="user exists"), 409 - token = secrets.token_urlsafe(32) - # The claim IS the user record: warmPoolRef only (env/VCTs would be - # rejected by the template policies), token hash as an annotation. +def parse_dsh_launch_token(logs: str) -> str | None: + """Return the token from the last ``dsh web: http://...?token=...`` line. + + DSH prints the launch URL once at startup. We scan the tail of the pod log + and keep the last match so a restarted process (which prints a new token) + wins over the stale one. + """ + token = None + for line in (logs or "").splitlines(): + match = _DSH_WEB_LINE.search(line) + if not match: + continue + values = parse_qs(urlsplit(match.group("url")).query).get("token") + if values and values[0]: + token = values[0] + return token + + +def extract_dsh_cookies(set_cookie_headers: list[str]) -> str: + """Join the ``dsh-auth-*`` pairs from Set-Cookie headers into a Cookie value.""" + pairs = [] + for raw in set_cookie_headers: + first = raw.split(";", 1)[0] + name, sep, value = first.partition("=") + name = name.strip() + if sep and name.startswith("dsh-auth-"): + pairs.append(f"{name}={value.strip()}") + return "; ".join(pairs) + + +def same_origin(headers) -> bool: + """True when Origin (or Referer) names the same authority as Host.""" + host = (headers.get("host") or "").strip().lower() + origin = headers.get("origin") + if origin: + return urlsplit(origin).netloc.lower() == host + referer = headers.get("referer") + if referer: + return urlsplit(referer).netloc.lower() == host + return False + + +def rewrite_location(location: str, user: str) -> str: + """Prefix absolute-path redirects so the browser stays under ``/u//``.""" + if not location.startswith("/") or location.startswith("//"): + return location + parts = urlsplit(location) + prefix = f"/u/{user}" + if parts.path == prefix or parts.path.startswith(f"{prefix}/"): + return location + query = [ + (key, value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + if key != "token" + ] + return urlunsplit(("", "", prefix + parts.path, urlencode(query), parts.fragment)) + + +def filtered_query(query_string: str) -> str: + """Drop ``?token=`` before a request is sent upstream.""" + pairs = [ + (key, value) + for key, value in parse_qsl(query_string, keep_blank_values=True) + if key != "token" + ] + return urlencode(pairs) + + +def _env_int(name: str, default: int) -> int: try: - crd.create_namespaced_custom_object( - GROUP, VERSION, NAMESPACE, "sandboxclaims", { - "apiVersion": f"{GROUP}/{VERSION}", - "kind": "SandboxClaim", - "metadata": { - "name": claim_name(user), - "annotations": { - TOKEN_ANNOTATION: hashlib.sha256(token.encode()).hexdigest()}, - }, - "spec": { - "warmPoolRef": {"name": POOL}, - "additionalPodMetadata": { - "labels": {"sandbox.users.io/dsh-user": user}}, - }, - }) - except client.ApiException as e: - if e.status == 409: # lost a concurrent-signup race for this user - return jsonify(error="user exists"), 409 - raise - sandbox = wait_ready(user, WAKE_TIMEOUT) - last_activity[user] = time.time() - return jsonify( - user=user, - token=token, # shown once; only the hash is stored - state=derive_state(sandbox), - sandbox=(sandbox or {}).get("metadata", {}).get("name"), - ), 201 + return int(os.environ.get(name, default)) + except ValueError: + return default -@app.get("/users/") -def get_user(user): - claim = get_claim(user) - if claim is None: - return jsonify(error="not found"), 404 - if not authorized(claim): # only the token holder may read their state - return jsonify(error="unauthorized"), 401 - sandbox = sandbox_of(claim) - return jsonify(user=user, state=derive_state(sandbox), - sandbox=(sandbox or {}).get("metadata", {}).get("name")) +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} -@app.delete("/users/") -def delete_user(user): - claim = get_claim(user) - if claim is None: - return jsonify(error="not found"), 404 - if not authorized(claim): # deletion cascades the PVC: token required - return jsonify(error="unauthorized"), 401 +def _env_float(name: str, default: float) -> float: try: - crd.delete_namespaced_custom_object( - GROUP, VERSION, NAMESPACE, "sandboxclaims", claim_name(user)) - except client.ApiException as e: - if e.status != 404: # already gone => idempotent success - raise - last_activity.pop(user, None) - return jsonify(user=user, note="claim deleted; sandbox and PVC cascade") + return float(os.environ.get(name, default)) + except ValueError: + return default -@app.route("/u//", defaults={"path": ""}, - methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) -@app.route("/u//", - methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) -def proxy(user, path): - claim = get_claim(user) - if claim is None: - return jsonify(error="unknown user"), 404 - if not authorized(claim): - return jsonify(error="unauthorized"), 401 +# --- Settings -------------------------------------------------------------- - # Reserve activity the moment the request is admitted — BEFORE the - # sandbox is resolved or woken — so the sweeper can never suspend - # between admission and the upstream dial. Every early return releases - # the reservation; the streaming path hands it to relay(), which - # releases it when the response finishes (or the client disconnects). - with flight_lock: - in_flight[user] = in_flight.get(user, 0) + 1 - last_activity[user] = time.time() - streaming = False - try: - sandbox = sandbox_of(claim) - if sandbox is None: - return jsonify(error="sandbox not provisioned yet"), 503 - if derive_state(sandbox) != "Ready": - # Wake-on-connect: flip the mode (safe in ANY state, including - # mid-suspension) and HOLD the request until Ready. - set_operating_mode(sandbox["metadata"]["name"], "Running") - sandbox = wait_ready(user, WAKE_TIMEOUT) - if sandbox is None: - return jsonify(error="DSH is waking up, retry shortly"), \ - 503, {"Retry-After": "10"} - last_activity[user] = time.time() - fqdn = (sandbox.get("status") or {}).get("serviceFQDN") - # Forward the original Host + X-Forwarded-* so DSH generates - # reachable links; still never forward the gateway's bearer token or - # a stray ?token= upstream where the app (or its logs) would see it. - headers = {k: v for k, v in request.headers - if k.lower() != "authorization"} - headers.setdefault("X-Forwarded-For", request.remote_addr or "") - headers.setdefault("X-Forwarded-Proto", request.scheme) - headers.setdefault("X-Forwarded-Host", request.host) - params = [(k, v) for k, v in request.args.items(multi=True) - if k != "token"] - upstream = f"http://{fqdn}:3080/{path}" +@dataclass(frozen=True) +class Settings: + """Runtime configuration, read from the environment by default. - # The pod can be Ready a beat before the server binds; retry the - # dial. - for attempt in range(5): + Defaults are the demo values from the example manifests. Everything is + injected into ``create_app`` so tests never need the environment or a + cluster. + """ + + namespace: str = "dsh-demo" + pool: str = "dsh-pool" + idle_timeout: int = 60 + wake_timeout: int = 120 + dsh_port: int = 3080 + dsh_container: str = "dsh-web" + dsh_log_tail_lines: int = 2000 + cookie_name: str = "dsh_gw_session" + cookie_secure: bool = False + cookie_max_age: int = 7 * 24 * 60 * 60 + max_body_bytes: int = 32 * 1024 * 1024 + http_timeout: float = 300.0 + sweep_interval: float = 15.0 + # Connection retries while a Ready pod is still binding its port. + upstream_attempts: int = 3 + upstream_retry_delay: float = 1.0 + + @classmethod + def from_env(cls) -> Settings: + return cls( + namespace=os.environ.get("NAMESPACE", "dsh-demo"), + pool=os.environ.get("POOL", "dsh-pool"), + idle_timeout=_env_int("IDLE_TIMEOUT", 60), + wake_timeout=_env_int("WAKE_TIMEOUT", 120), + dsh_port=_env_int("DSH_PORT", 3080), + dsh_container=os.environ.get("DSH_CONTAINER", "dsh-web"), + dsh_log_tail_lines=_env_int("DSH_LOG_TAIL_LINES", 2000), + cookie_name=os.environ.get("COOKIE_NAME", "dsh_gw_session"), + cookie_secure=_env_bool("COOKIE_SECURE", False), + cookie_max_age=_env_int("COOKIE_MAX_AGE", 7 * 24 * 60 * 60), + max_body_bytes=_env_int("MAX_BODY_MB", 32) * 1024 * 1024, + http_timeout=_env_float("HTTP_TIMEOUT", 300.0), + sweep_interval=_env_float("SWEEP_INTERVAL", 15.0), + ) + + +# --- Kubernetes adapter ---------------------------------------------------- +# +# The official Kubernetes client is synchronous. Every call is pushed onto a +# worker thread with anyio.to_thread so it never blocks the event loop. The +# client and its credentials are loaded lazily, on the first real call: that +# keeps ``GET /healthz`` (and importing this module) cluster-free. + + +class Kubernetes: + """Thin async wrapper over the sync CustomObjects/CoreV1 API clients.""" + + def __init__(self, namespace: str = "dsh-demo"): + self.namespace = namespace + self._crd: k8s_client.CustomObjectsApi | None = None + self._core: k8s_client.CoreV1Api | None = None + + def _clients(self): + if self._crd is None: try: - r = requests.request( - request.method, upstream, params=params, - data=request.get_data(), headers=headers, - stream=True, timeout=300) - break - except requests.ConnectionError: - if attempt == 4: - return jsonify(error="upstream unavailable"), 502 - time.sleep(1) + k8s_config.load_incluster_config() + except k8s_config.ConfigException: + k8s_config.load_kube_config() + self._crd = k8s_client.CustomObjectsApi() + self._core = k8s_client.CoreV1Api() + return self._crd, self._core - def relay(): - try: - yield from r.iter_content(chunk_size=None) - finally: # runs on stream completion AND client disconnect - r.close() # release the upstream connection either way - end_flight(user) + async def _call(self, which: str, method: str, *args, **kwargs): + def op(): + crd, core = self._clients() + target = crd if which == "crd" else core + return getattr(target, method)(*args, **kwargs) - streaming = True # relay() now owns the in-flight reservation - return Response( - relay(), - status=r.status_code, - headers=[(k, v) for k, v in r.headers.items() - if k.lower() not in ("transfer-encoding", "connection")]) - finally: - if not streaming: - end_flight(user) + return await anyio.to_thread.run_sync(op) + # -- claims ------------------------------------------------------------ -def idle_sweeper(): - """Suspend sandboxes idle for IDLE_TIMEOUT (pod deleted, PVC survives).""" - while True: - time.sleep(15) + async def get_claim(self, user: str) -> dict | None: try: - claims = crd.list_namespaced_custom_object( - GROUP, VERSION, NAMESPACE, "sandboxclaims")["items"] - for c in claims: - user = c["metadata"]["name"].removeprefix("dsh-") - try: - sandbox = sandbox_of(c) - except client.ApiException: - continue - if sandbox is None or derive_state(sandbox) != "Ready": - continue - # Check-and-suspend atomically with proxy admission (which - # increments in_flight under this same lock). - with flight_lock: - if in_flight.get(user, 0) > 0: - continue # requests in flight => the user is active - idle = time.time() - \ - last_activity.setdefault(user, time.time()) - if idle < IDLE_TIMEOUT: - continue - print(f"suspending {user} (idle {int(idle)}s)", - flush=True) - set_operating_mode(sandbox["metadata"]["name"], - "Suspended") - except Exception as e: - # A transient API error must not kill this daemon thread — that - # would silently stop idle suspension for good. - print(f"sweeper: transient error, retrying next cycle: {e}", - flush=True) + return await self._call( + "crd", + "get_namespaced_custom_object", + GROUP, + VERSION, + self.namespace, + "sandboxclaims", + claim_name(user), + ) + except ApiException as exc: + if exc.status == 404: + return None + raise + + async def create_claim(self, user: str, token_hash: str, pool: str) -> None: + body = { + "apiVersion": f"{GROUP}/{VERSION}", + "kind": "SandboxClaim", + "metadata": { + "name": claim_name(user), + "annotations": {TOKEN_ANNOTATION: token_hash}, + }, + "spec": { + "warmPoolRef": {"name": pool}, + "additionalPodMetadata": {"labels": {USER_LABEL: user}}, + }, + } + try: + await self._call( + "crd", + "create_namespaced_custom_object", + GROUP, + VERSION, + self.namespace, + "sandboxclaims", + body, + ) + except ApiException as exc: + if exc.status == 409: + raise UserExists(user) from exc + raise + + async def delete_claim(self, user: str) -> None: + try: + await self._call( + "crd", + "delete_namespaced_custom_object", + GROUP, + VERSION, + self.namespace, + "sandboxclaims", + claim_name(user), + ) + except ApiException as exc: + if exc.status != 404: # already gone => idempotent success + raise + + async def list_claims(self) -> list[dict]: + result = await self._call( + "crd", + "list_namespaced_custom_object", + GROUP, + VERSION, + self.namespace, + "sandboxclaims", + ) + return result.get("items", []) + + # -- sandboxes --------------------------------------------------------- + + async def get_sandbox(self, name: str) -> dict | None: + try: + return await self._call( + "crd", + "get_namespaced_custom_object", + CORE_GROUP, + VERSION, + self.namespace, + "sandboxes", + name, + ) + except ApiException as exc: + if exc.status == 404: + return None + raise + + async def patch_operating_mode(self, name: str, mode: str) -> None: + await self._call( + "crd", + "patch_namespaced_custom_object", + CORE_GROUP, + VERSION, + self.namespace, + "sandboxes", + name, + {"spec": {"operatingMode": mode}}, + ) + + # -- pods and logs ----------------------------------------------------- + + async def get_pod(self, name: str) -> dict | None: + try: + return await self._call( + "core", "read_namespaced_pod", name, self.namespace + ) + except ApiException as exc: + if exc.status == 404: + return None + raise + + async def read_pod_logs( + self, name: str, container: str, tail_lines: int + ) -> str: + result = await self._call( + "core", + "read_namespaced_pod_log", + name, + self.namespace, + container=container, + tail_lines=tail_lines, + ) + return result or "" -if __name__ == "__main__": - threading.Thread(target=idle_sweeper, daemon=True).start() - # threaded=True so one user's wake-on-connect (which can hold a request - # for up to WAKE_TIMEOUT) doesn't block everyone else. Still Werkzeug's - # dev server — a real deployment would sit behind gunicorn or similar. - app.run(host="0.0.0.0", port=8080, threaded=True) +# --- Gateway --------------------------------------------------------------- + + +@dataclass +class GatewayState: + """In-memory, per-process state (single replica by design).""" + + last_activity: dict[str, float] = field(default_factory=dict) + in_flight: dict[str, int] = field(default_factory=dict) + maintenance: dict[str, bool] = field(default_factory=dict) + # (user, pod UID, browser Host) -> "dsh-auth-a=...; dsh-auth-b=..." + handshake: dict[tuple[str, str, str], str] = field(default_factory=dict) + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + +class Gateway: + """Route handlers plus the state machine they share.""" + + def __init__( + self, + settings: Settings, + k8s, + http_client: httpx.AsyncClient, + ws_connect, + ): + self.settings = settings + self.k8s = k8s + self.http = http_client + self.ws_connect = ws_connect + self.state = GatewayState() + + # -- credentials ------------------------------------------------------- + + def credentials(self, headers, cookies) -> tuple[str | None, str | None]: + """Return ``(token, cookie_user)``; bearer wins over the session cookie.""" + auth = headers.get("authorization", "") + if auth.lower().startswith("bearer "): + return auth[7:].strip(), None + raw = cookies.get(self.settings.cookie_name) + if raw: + user, sep, token = raw.partition(":") + if sep and user and token: + return token, user + return None, None + + def _authorized( + self, + claim: dict, + token: str | None, + cookie_user: str | None, + user: str, + ) -> bool: + if cookie_user is not None and cookie_user != user: + return False # a session for another user never authorizes this one + return token_matches(claim, token) + + @staticmethod + def _mutation_from_cookie(headers, method: str, cookie_user: str | None) -> bool: + return ( + cookie_user is not None + and method.upper() not in SAFE_METHODS + and not same_origin(headers) + ) + + def set_session(self, response: Response, user: str, token: str) -> None: + response.set_cookie( + self.settings.cookie_name, + f"{user}:{token}", + max_age=self.settings.cookie_max_age, + path="/", + httponly=True, + samesite="lax", + secure=self.settings.cookie_secure, + ) + + # -- activity / maintenance ------------------------------------------- + + async def _touch(self, user: str) -> None: + async with self.state.lock: + self.state.last_activity[user] = time.time() + + async def _acquire(self, user: str) -> bool: + """Reserve an in-flight slot, waiting out a concurrent suspend.""" + deadline = time.monotonic() + self.settings.wake_timeout + while True: + async with self.state.lock: + if not self.state.maintenance.get(user): + self.state.in_flight[user] = ( + self.state.in_flight.get(user, 0) + 1 + ) + self.state.last_activity[user] = time.time() + return True + if time.monotonic() > deadline: + return False + await asyncio.sleep(0.1) + + async def _release(self, user: str) -> None: + async with self.state.lock: + self.state.in_flight[user] = max( + self.state.in_flight.get(user, 1) - 1, 0 + ) + self.state.last_activity[user] = time.time() + + # -- sandbox / handshake helpers -------------------------------------- + + async def _sandbox_of(self, claim: dict) -> dict | None: + name = (claim.get("status") or {}).get("sandbox", {}).get("name") + if not name: + return None + return await self.k8s.get_sandbox(name) + + async def wait_ready(self, user: str, timeout: int) -> dict | None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + claim = await self.k8s.get_claim(user) + sandbox = await self._sandbox_of(claim) if claim else None + if sandbox is not None and derive_state(sandbox) == "Ready": + return sandbox + await asyncio.sleep(1) + return None + + async def _resolve_sandbox(self, user: str, claim: dict) -> tuple[dict, str]: + sandbox = await self._sandbox_of(claim) + if sandbox is None: + raise SandboxUnavailable("sandbox not provisioned yet") + if derive_state(sandbox) != "Ready": + # Wake-on-connect: safe in ANY state, including mid-suspension. + self._clear_handshake(user) + try: + await self.k8s.patch_operating_mode( + sandbox["metadata"]["name"], "Running" + ) + except Exception as exc: + raise SandboxUnavailable("could not wake the sandbox") from exc + sandbox = await self.wait_ready(user, self.settings.wake_timeout) + if sandbox is None: + raise SandboxUnavailable("DSH is waking up, retry shortly") + await self._touch(user) + fqdn = (sandbox.get("status") or {}).get("serviceFQDN") + if not fqdn: + raise SandboxUnavailable("sandbox has no service FQDN yet") + return sandbox, fqdn + + def _pod_name(self, sandbox: dict) -> str: + annotations = sandbox.get("metadata", {}).get("annotations") or {} + return ( + annotations.get(POD_NAME_ANNOTATION) + or sandbox["metadata"]["name"] + ) + + def _clear_handshake(self, user: str) -> None: + for key in [k for k in self.state.handshake if k[0] == user]: + self.state.handshake.pop(key, None) + + async def dsh_cookie( + self, + user: str, + sandbox: dict, + fqdn: str, + host: str, + force: bool = False, + ) -> tuple[str, tuple[str, str, str], bool]: + """Return ``(cookie, cache_key, was_cached)``, handshaking on a miss.""" + pod_name = self._pod_name(sandbox) + try: + pod = await self.k8s.get_pod(pod_name) + except ApiException as exc: + raise HandshakeError(f"could not read pod {pod_name}: {exc}") from exc + if pod is None: + raise HandshakeError(f"pod {pod_name} not found") + uid = (pod.get("metadata") or {}).get("uid") or pod_name + key = (user, uid, host) + if not force: + cached = self.state.handshake.get(key) + if cached: + return cached, key, True + try: + logs = await self.k8s.read_pod_logs( + pod_name, + self.settings.dsh_container, + self.settings.dsh_log_tail_lines, + ) + except ApiException as exc: + raise HandshakeError( + f"could not read logs for {pod_name}: {exc}" + ) from exc + launch_token = parse_dsh_launch_token(logs) + if not launch_token: + raise HandshakeError("DSH launch token not found in pod logs") + url = ( + f"http://{fqdn}:{self.settings.dsh_port}/" + f"?token={quote(launch_token, safe='')}" + ) + headers = { + "Host": host, + "X-Forwarded-Host": host, + "X-Forwarded-Proto": "http", + } + try: + response = await self.http.get( + url, + headers=headers, + follow_redirects=False, + timeout=self.settings.http_timeout, + ) + except httpx.HTTPError as exc: + raise HandshakeError(f"DSH handshake failed: {exc}") from exc + if response.status_code >= 400: + raise HandshakeError( + f"DSH handshake returned {response.status_code}" + ) + cookie = extract_dsh_cookies(response.headers.get_list("set-cookie")) + if not cookie: + raise HandshakeError("DSH handshake set no dsh-auth cookie") + self.state.handshake[key] = cookie + return cookie, key, False + + # -- landing page / health -------------------------------------------- + + async def redirect_to_workspace(self, request: Request) -> Response: + """``/u/`` -> ``/u//``; DSH's relative URLs need the slash.""" + user = request.path_params["user"] + if not valid_user(user): + return self._error(404, "unknown user") + target = f"/u/{user}/" + if request.url.query: + target = f"{target}?{request.url.query}" + return RedirectResponse(target, status_code=308) + + async def healthz(self, request: Request) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + async def landing(self, request: Request) -> HTMLResponse: + token, user = self.credentials(request.headers, request.cookies) + session = "" + if user and token: + safe_user = html.escape(user) + session = ( + f'

Signed in as {safe_user}. ' + f'Open workspace

' + '
' + '
' + ) + body = f""" + + + + +DSH sandbox gateway + + + +

DeepSeek Harness sandboxes

+{session} +
+

Create a sandbox

+
+ + +
+
+
+

Sign in with a token

+
+ + + +
+
+

Automation can use Authorization: Bearer <token> +against the JSON API. The browser session is an HttpOnly cookie.

+ +""" + return HTMLResponse(body) + + # -- JSON helpers ------------------------------------------------------ + + @staticmethod + def _error(status_code: int, message: str, headers=None) -> JSONResponse: + return JSONResponse({"error": message}, status_code=status_code, headers=headers) + + @staticmethod + def _wants_form(request: Request) -> bool: + ctype = request.headers.get("content-type", "").lower() + return ctype.startswith( + ("application/x-www-form-urlencoded", "multipart/form-data") + ) + + async def _read_payload(self, request: Request) -> dict: + if request.headers.get("content-type", "").lower().startswith( + "application/json" + ): + data = await self._read_body(request) + try: + payload = json.loads(data) + except ValueError: + payload = None + return payload if isinstance(payload, dict) else {} + form = await request.form() + return {key: form.get(key) for key in form} + + async def _read_body(self, request: Request) -> bytes: + limit = self.settings.max_body_bytes + raw_length = request.headers.get("content-length") + if raw_length: + try: + if int(raw_length) > limit: + raise BodyTooLarge + except ValueError: + pass + chunks, total = [], 0 + async for chunk in request.stream(): + total += len(chunk) + if total > limit: + raise BodyTooLarge + chunks.append(chunk) + return b"".join(chunks) + + # -- users ------------------------------------------------------------- + + async def signup(self, request: Request) -> Response: + wants_form = self._wants_form(request) + try: + payload = await self._read_payload(request) + except BodyTooLarge: + return self._error(413, "request body too large") + user = payload.get("user", "") + if not valid_user(user): + return self._error( + 400, "body must be {'user': ''}" + ) + if await self.k8s.get_claim(user) is not None: + return self._error(409, "user exists") + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode()).hexdigest() + try: + await self.k8s.create_claim(user, token_hash, self.settings.pool) + except UserExists: + return self._error(409, "user exists") + await self._touch(user) + sandbox = await self.wait_ready(user, self.settings.wake_timeout) + state = derive_state(sandbox) + sandbox_name = (sandbox or {}).get("metadata", {}).get("name") + if wants_form: + response = RedirectResponse(f"/u/{user}/", status_code=303) + self.set_session(response, user, token) + return response + response = JSONResponse( + {"user": user, "token": token, "state": state, "sandbox": sandbox_name}, + status_code=201 if state == "Ready" else 202, + ) + self.set_session(response, user, token) + return response + + async def login(self, request: Request) -> Response: + wants_form = self._wants_form(request) + try: + payload = await self._read_payload(request) + except BodyTooLarge: + return self._error(413, "request body too large") + user = payload.get("user", "") + token = payload.get("token", "") + if not isinstance(user, str) or not isinstance(token, str): + return self._error(400, "user and token are required") + claim = await self.k8s.get_claim(user) + if claim is None: + return self._error(404, "not found") + if not token_matches(claim, token): + return self._error(401, "unauthorized") + await self._touch(user) + if wants_form: + response = RedirectResponse(f"/u/{user}/", status_code=303) + else: + response = JSONResponse({"user": user, "state": "ok"}) + self.set_session(response, user, token) + return response + + async def logout(self, request: Request) -> Response: + response = RedirectResponse("/", status_code=303) + response.delete_cookie(self.settings.cookie_name, path="/") + return response + + async def get_user(self, request: Request) -> Response: + user = request.path_params["user"] + if not valid_user(user): + return self._error(404, "not found") + claim = await self.k8s.get_claim(user) + if claim is None: + return self._error(404, "not found") + token, cookie_user = self.credentials(request.headers, request.cookies) + if not self._authorized(claim, token, cookie_user, user): + return self._error(401, "unauthorized") + sandbox = await self._sandbox_of(claim) + return JSONResponse( + { + "user": user, + "state": derive_state(sandbox), + "sandbox": (sandbox or {}).get("metadata", {}).get("name"), + } + ) + + async def delete_user(self, request: Request) -> Response: + user = request.path_params["user"] + if not valid_user(user): + return self._error(404, "not found") + claim = await self.k8s.get_claim(user) + if claim is None: + return self._error(404, "not found") + token, cookie_user = self.credentials(request.headers, request.cookies) + if not self._authorized(claim, token, cookie_user, user): + return self._error(401, "unauthorized") + if self._mutation_from_cookie(request.headers, "DELETE", cookie_user): + return self._error(403, "cross-origin request rejected") + await self.k8s.delete_claim(user) + self._clear_handshake(user) + async with self.state.lock: + self.state.last_activity.pop(user, None) + self.state.in_flight.pop(user, None) + return JSONResponse( + {"user": user, "note": "claim deleted; sandbox and PVC cascade"} + ) + + # -- HTTP proxy -------------------------------------------------------- + + async def proxy_http(self, request: Request) -> Response: + user = request.path_params["user"] + if not valid_user(user): + return self._error(404, "unknown user") + claim = await self.k8s.get_claim(user) + if claim is None: + return self._error(404, "unknown user") + token, cookie_user = self.credentials(request.headers, request.cookies) + if not self._authorized(claim, token, cookie_user, user): + return self._error(401, "unauthorized") + if self._mutation_from_cookie(request.headers, request.method, cookie_user): + return self._error(403, "cross-origin request rejected") + + if not await self._acquire(user): + return self._error( + 503, "gateway busy, retry shortly", {"Retry-After": "5"} + ) + streaming = False + try: + sandbox, fqdn = await self._resolve_sandbox(user, claim) + host = request.headers.get("host", "") + cookie, _, was_cached = await self.dsh_cookie( + user, sandbox, fqdn, host + ) + body = await self._read_body(request) + + upstream = await self._send_upstream( + request, user, fqdn, cookie, body + ) + if upstream.status_code == 401 and was_cached: + # The cached DSH cookie expired (pod restart, logout, ...): + # handshake once more and replay the (already buffered) request. + await upstream.aclose() + cookie, _, _ = await self.dsh_cookie( + user, sandbox, fqdn, host, force=True + ) + upstream = await self._send_upstream( + request, user, fqdn, cookie, body + ) + + response = StreamingResponse( + self._relay(user, upstream), + status_code=upstream.status_code, + headers=self._response_headers(upstream, user), + ) + streaming = True + return response + except BodyTooLarge: + return self._error(413, "request body too large") + except (HandshakeError, SandboxUnavailable) as exc: + return self._error( + 503, + f"{exc}; retry shortly", + {"Retry-After": "10"}, + ) + except UpstreamError: + return self._error(502, "upstream unavailable") + finally: + if not streaming: + await self._release(user) + + async def _send_upstream( + self, + request: Request, + user: str, + fqdn: str, + cookie: str, + body: bytes, + ) -> httpx.Response: + path = request.path_params.get("path", "") + url = f"http://{fqdn}:{self.settings.dsh_port}/{path}" + query = filtered_query(request.url.query) + if query: + url = f"{url}?{query}" + + headers = { + key: value + for key, value in request.headers.items() + if key.lower() not in OWNED_REQUEST_HEADERS + and key.lower() != "content-length" + } + host = request.headers.get("host", "") + headers["host"] = host + headers["cookie"] = cookie # only the cached DSH auth cookie + headers["x-forwarded-host"] = host + headers["x-forwarded-proto"] = request.url.scheme + if request.client: + headers["x-forwarded-for"] = request.client.host + else: + headers["x-forwarded-for"] = "" + port = host.rsplit(":", 1)[1] if ":" in host else "" + headers["x-forwarded-port"] = port or ( + "443" if request.url.scheme == "https" else "80" + ) + + last_error: Exception | None = None + for attempt in range(self.settings.upstream_attempts): + try: + upstream_request = self.http.build_request( + request.method, url, headers=headers, content=body + ) + return await self.http.send(upstream_request, stream=True) + except (httpx.ConnectError, httpx.ConnectTimeout) as exc: + # Retry only failures that cannot have reached the upstream; + # replaying a timed-out POST could duplicate a mutation. + last_error = exc + if attempt + 1 < self.settings.upstream_attempts: + await asyncio.sleep(self.settings.upstream_retry_delay) + except httpx.HTTPError as exc: + raise UpstreamError("upstream request failed") from exc + raise UpstreamError("upstream unavailable") from last_error + + @staticmethod + def _response_headers(upstream: httpx.Response, user: str): + headers = [] + for key, value in upstream.headers.multi_items(): + lowered = key.lower() + if lowered in HOP_BY_HOP or lowered == "set-cookie": + continue # never leak upstream cookies to the browser + if lowered == "location": + value = rewrite_location(value, user) + headers.append((key, value)) + return Headers(raw=[(k.lower().encode("latin-1"), v.encode("latin-1")) for k, v in headers]) + + async def _relay(self, user: str, upstream: httpx.Response): + try: + async for chunk in upstream.aiter_raw(): + yield chunk + finally: + with contextlib.suppress(Exception): + await upstream.aclose() + await self._release(user) + + # -- WebSocket proxy --------------------------------------------------- + + async def _ws_reject(self, websocket: WebSocket, code: int) -> None: + # A close code can only be delivered after the upgrade is accepted; + # before accept Starlette would turn this into an HTTP 403. + with contextlib.suppress(Exception): + await websocket.accept() + with contextlib.suppress(Exception): + await websocket.close(code=code) + + async def proxy_ws(self, websocket: WebSocket) -> None: + user = websocket.path_params["user"] + if not valid_user(user): + return await self._ws_reject(websocket, 4401) + claim = await self.k8s.get_claim(user) + if claim is None: + return await self._ws_reject(websocket, 4401) + token, cookie_user = self.credentials(websocket.headers, websocket.cookies) + if not self._authorized(claim, token, cookie_user, user): + return await self._ws_reject(websocket, 4401) + if cookie_user is not None and not same_origin(websocket.headers): + return await self._ws_reject(websocket, 4401) + + if not await self._acquire(user): + return await self._ws_reject(websocket, 1011) + upstream_cm = None + try: + try: + sandbox, fqdn = await self._resolve_sandbox(user, claim) + host = websocket.headers.get("host", "") + cookie, _, _ = await self.dsh_cookie(user, sandbox, fqdn, host) + path = websocket.path_params.get("path", "") + url = f"ws://{fqdn}:{self.settings.dsh_port}/{path}" + query = filtered_query(websocket.url.query) + if query: + url = f"{url}?{query}" + headers = { + key: value + for key, value in websocket.headers.items() + if key.lower() not in OWNED_REQUEST_HEADERS + and not key.lower().startswith("sec-websocket-") + } + headers["host"] = host + headers["cookie"] = cookie + headers["x-forwarded-host"] = host + headers["x-forwarded-proto"] = websocket.url.scheme + if websocket.client: + headers["x-forwarded-for"] = websocket.client.host + subprotocols = [ + part.strip() + for part in websocket.headers.get( + "sec-websocket-protocol", "" + ).split(",") + if part.strip() + ] + upstream_cm = self.ws_connect( + url, + additional_headers=headers, + subprotocols=subprotocols or None, + open_timeout=self.settings.wake_timeout, + max_size=None, + proxy=None, + ) + upstream = await upstream_cm.__aenter__() + except Exception: + return await self._ws_reject(websocket, 1011) + + try: + await websocket.accept( + subprotocol=getattr(upstream, "subprotocol", None) + ) + except RuntimeError: + pass + await self._pump(websocket, upstream) + with contextlib.suppress(Exception): + await websocket.close( + code=self._upstream_close_code(upstream) + ) + finally: + if upstream_cm is not None: + with contextlib.suppress(Exception): + await upstream_cm.__aexit__(None, None, None) + await self._release(user) + + @staticmethod + def _upstream_close_code(upstream) -> int: + code = getattr(upstream, "close_code", None) + if isinstance(code, int) and 1000 <= code <= 4999 and code not in (1005, 1006): + return code + return 1000 + + async def _pump(self, websocket: WebSocket, upstream) -> None: + async def client_to_upstream(): + try: + while True: + message = await websocket.receive() + if message["type"] == "websocket.disconnect": + return + if message.get("text") is not None: + await upstream.send(message["text"]) + elif message.get("bytes") is not None: + await upstream.send(message["bytes"]) + finally: + with contextlib.suppress(Exception): + await upstream.close() + + async def upstream_to_client(): + async for message in upstream: + if isinstance(message, str): + await websocket.send_text(message) + else: + await websocket.send_bytes(message) + + tasks = [ + asyncio.create_task(client_to_upstream()), + asyncio.create_task(upstream_to_client()), + ] + done, pending = await asyncio.wait( + tasks, return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + await asyncio.gather(*done, *pending, return_exceptions=True) + + # -- idle sweeper ------------------------------------------------------ + + async def sweep_once(self) -> None: + claims = await self.k8s.list_claims() + for claim in claims: + user = claim_user(claim) + try: + sandbox = await self._sandbox_of(claim) + except ApiException: + continue # transient/stale claim; retry next cycle + if sandbox is None or derive_state(sandbox) != "Ready": + continue + async with self.state.lock: + if self.state.maintenance.get(user): + continue + if self.state.in_flight.get(user, 0) > 0: + continue # requests in flight => the user is active + idle = time.time() - self.state.last_activity.setdefault( + user, time.time() + ) + if idle < self.settings.idle_timeout: + continue + self.state.maintenance[user] = True + try: + print(f"suspending {user} (idle {int(idle)}s)", flush=True) + await self.k8s.patch_operating_mode( + sandbox["metadata"]["name"], "Suspended" + ) + self._clear_handshake(user) + finally: + async with self.state.lock: + self.state.maintenance[user] = False + + async def sweeper_loop(self) -> None: + while True: + await asyncio.sleep(self.settings.sweep_interval) + try: + await self.sweep_once() + except asyncio.CancelledError: + raise + except Exception as exc: + print( + f"sweeper: transient error, retrying next cycle: {exc}", + flush=True, + ) + + +# --- Application factory --------------------------------------------------- + + +def _lifespan(gateway: Gateway, close_http_client: httpx.AsyncClient | None = None): + @contextlib.asynccontextmanager + async def lifespan(app): + task = asyncio.create_task(gateway.sweeper_loop()) + try: + yield + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + if close_http_client is not None: + with contextlib.suppress(Exception): + await close_http_client.aclose() + + return lifespan + + +def create_app( + settings: Settings | None = None, + k8s=None, + http_client: httpx.AsyncClient | None = None, + ws_connect=None, +) -> Starlette: + """Build the ASGI app. All collaborators are injectable for tests.""" + settings = settings or Settings.from_env() + k8s = k8s if k8s is not None else Kubernetes(settings.namespace) + owns_http_client = http_client is None + if http_client is None: + http_client = httpx.AsyncClient( + follow_redirects=False, timeout=settings.http_timeout + ) + gateway = Gateway( + settings, k8s, http_client, ws_connect or websockets.connect + ) + routes = [ + Route("/healthz", gateway.healthz), + Route("/", gateway.landing), + Route("/users", gateway.signup, methods=["POST"]), + Route("/login", gateway.login, methods=["POST"]), + Route("/logout", gateway.logout, methods=["POST"]), + Route("/users/{user}", gateway.get_user), + Route("/users/{user}", gateway.delete_user, methods=["DELETE"]), + Route("/u/{user}", gateway.redirect_to_workspace), + Route( + "/u/{user}/{path:path}", + gateway.proxy_http, + methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], + ), + WebSocketRoute("/u/{user}/{path:path}", gateway.proxy_ws), + ] + app = Starlette( + routes=routes, + lifespan=_lifespan(gateway, http_client if owns_http_client else None), + ) + app.state.gateway = gateway + return app diff --git a/examples/dsh-agents-as-a-service/gateway/pyproject.toml b/examples/dsh-agents-as-a-service/gateway/pyproject.toml new file mode 100644 index 0000000..de23414 --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/pyproject.toml @@ -0,0 +1,14 @@ +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["."] + +[tool.ruff] +target-version = "py312" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "PIE", "RUF"] +ignore = ["RUF001", "RUF002", "RUF003"] + +[tool.ruff.lint.per-file-ignores] +"test_gateway.py" = ["E501"] diff --git a/examples/dsh-agents-as-a-service/gateway/requirements-dev.txt b/examples/dsh-agents-as-a-service/gateway/requirements-dev.txt new file mode 100644 index 0000000..5c51b9d --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt +pytest==9.1.1 +pytest-asyncio==1.4.0 +ruff==0.17.1 diff --git a/examples/dsh-agents-as-a-service/gateway/requirements.txt b/examples/dsh-agents-as-a-service/gateway/requirements.txt index ad59624..daf3ef1 100644 --- a/examples/dsh-agents-as-a-service/gateway/requirements.txt +++ b/examples/dsh-agents-as-a-service/gateway/requirements.txt @@ -1,3 +1,6 @@ -flask==3.1.* -requests==2.32.* -kubernetes==31.0.* +starlette==1.9.0 +uvicorn[standard]==0.54.0 +httpx==0.28.1 +websockets==17.3 +kubernetes==37.0.1 +python-multipart==0.0.32 diff --git a/examples/dsh-agents-as-a-service/gateway/test_gateway.py b/examples/dsh-agents-as-a-service/gateway/test_gateway.py index eba2a68..52330a8 100644 --- a/examples/dsh-agents-as-a-service/gateway/test_gateway.py +++ b/examples/dsh-agents-as-a-service/gateway/test_gateway.py @@ -1,382 +1,1240 @@ -""" -Unit tests for the dsh-agents-as-a-service gateway example. +"""Unit tests for the browser-usable DSH gateway. -gateway.py runs as the control+data plane in front of agent-sandbox; these -tests cover its security/parsing logic (bearer-token auth, DNS-1123 user -validation, the mode x conditions state machine) and the /users and -idle-sweeper decision logic, with the Kubernetes CustomObjectsApi mocked. -No kind cluster or container build is needed. - -gateway.py isn't a valid module name to import as a package member from a -test file that also wants a fresh module per test (its top level calls -config.load_incluster_config()/load_kube_config(), both of which would blow -up or reach a real cluster), so each test loads its own copy by file path -via the `gw` fixture, mirroring examples/hpa-swp-scaling/test_create_claim.py. +Everything here runs without a cluster: the Kubernetes adapter, the upstream +httpx client and the WebSocket connector are injected fakes. The HTTP tests +drive the real ASGI app through ``httpx.ASGITransport``; the WebSocket tests +use Starlette's ``TestClient`` (which runs the lifespan) against a fake +``ws_connect`` async context manager. """ +import asyncio import hashlib -import importlib.util import json -import os import time from types import SimpleNamespace -from unittest import mock +import httpx import pytest -from kubernetes import client as k8s_client -from kubernetes.config import ConfigException +from kubernetes.client.exceptions import ApiException +from starlette.testclient import TestClient +from starlette.websockets import WebSocketDisconnect -GATEWAY_PATH = os.path.join(os.path.dirname(__file__), "gateway.py") +from gateway import ( + Settings, + UserExists, + claim_name, + claim_user, + condition, + create_app, + derive_state, + extract_dsh_cookies, + filtered_query, + parse_dsh_launch_token, + rewrite_location, + same_origin, + token_matches, +) + +TOKEN = "s3cret-token" +POD_UID = "pod-uid-1" +FQDN = "dsh-alice-sbx.test-ns.svc.cluster.local" -class _StopLoop(Exception): - """Sentinel used to break idle_sweeper()'s `while True` after one pass.""" +def dsh_response(status=200, content=b"", headers=None): + """A MockTransport response that is a real stream (``aiter_raw`` works).""" + return httpx.Response( + status, headers=headers, stream=httpx.ByteStream(content) + ) -def _load_gateway(monkeypatch, mock_crd): - monkeypatch.setenv("NAMESPACE", "test-ns") - monkeypatch.setenv("POOL", "test-pool") - monkeypatch.setenv("IDLE_TIMEOUT", "60") - monkeypatch.setenv("WAKE_TIMEOUT", "120") +# --- fakes ----------------------------------------------------------------- - with mock.patch("kubernetes.config.load_incluster_config", - side_effect=ConfigException("not running in a cluster")), \ - mock.patch("kubernetes.config.load_kube_config"), \ - mock.patch("kubernetes.client.CustomObjectsApi", return_value=mock_crd): - spec = importlib.util.spec_from_file_location("dsh_gateway", GATEWAY_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + +class FakeK8s: + def __init__(self): + self.claims = {} + self.sandboxes = {} + self.pods = {} + self.logs = {} + self.created = [] + self.deleted = [] + self.patches = [] + self.conflict_on_create = False + self.get_claim_error = None + self.patch_error = None + + async def get_claim(self, user): + if self.get_claim_error is not None: + raise self.get_claim_error + return self.claims.get(user) + + async def create_claim(self, user, token_hash, pool): + if self.conflict_on_create: + raise UserExists(user) + self.created.append((user, token_hash, pool)) + self.claims.setdefault(user, {"metadata": {"name": claim_name(user)}}) + + async def delete_claim(self, user): + self.deleted.append(user) + self.claims.pop(user, None) + + async def list_claims(self): + return list(self.claims.values()) + + async def get_sandbox(self, name): + return self.sandboxes.get(name) + + async def patch_operating_mode(self, name, mode): + if self.patch_error is not None: + raise self.patch_error + self.patches.append((name, mode)) + + async def get_pod(self, name): + return self.pods.get(name) + + async def read_pod_logs(self, name, container, tail_lines): + return self.logs.get(name, "") + + +class FakeUpstream: + """An httpx.MockTransport handler for both the handshake and the proxy.""" + + def __init__(self): + self.requests = [] + self.handshake_requests = [] + self.handshake_status = 303 + self.handshake_cookies = [ + "dsh-auth-a=one; Path=/; HttpOnly; SameSite=Strict" + ] + self.proxy_handler = None + self.proxy_requests = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if request.url.path == "/" and "token=" in str(request.url.query): + self.handshake_requests.append(request) + cookies = self.handshake_cookies + index = min(len(self.handshake_requests) - 1, len(cookies) - 1) + return dsh_response( + self.handshake_status, + headers=[ + ("location", "/"), + ("set-cookie", cookies[index]), + ], + ) + self.proxy_requests.append(request) + if self.proxy_handler is not None: + return self.proxy_handler(request) + return dsh_response( + 200, b"hello from DSH", {"content-type": "text/plain"} + ) + + +class FakeWS: + """Async context manager matching websockets' ClientConnection subset.""" + + def __init__(self): + self.sent = [] + self.closed = False + self.close_code = None + self.subprotocol = None + self._stage = 0 + self._sent_event = asyncio.Event() + + async def send(self, data): + self.sent.append(data) + self._sent_event.set() + + async def close(self, code=1000): + self.closed = True + self.close_code = code + + def __aiter__(self): + return self + + async def __anext__(self): + if self._stage == 0: + self._stage = 1 + return "welcome" + if self._stage == 1: + await self._sent_event.wait() + self._stage = 2 + return "pong" + raise StopAsyncIteration + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + return False + + +class FakeWSConnect: + def __init__(self): + self.calls = [] + self.instances = [] + self.error = None + + def __call__(self, url, **kwargs): + self.calls.append((url, kwargs)) + if self.error is not None: + raise self.error + ws = FakeWS() + self.instances.append(ws) + return ws + + +def add_user(k8s, user="alice", token=TOKEN, ready=True, pod_name=None, suffix=""): + """Populate the fake API objects for a Ready (or Waking) user.""" + sandbox_name = f"dsh-{user}-sbx{suffix}" + pod_name = pod_name or sandbox_name + k8s.claims[user] = { + "metadata": { + "name": claim_name(user), + "annotations": { + "aaas.example.com/token-sha256": hashlib.sha256( + token.encode() + ).hexdigest() + }, + }, + "status": {"sandbox": {"name": sandbox_name}}, + } + k8s.sandboxes[sandbox_name] = { + "metadata": {"name": sandbox_name, "annotations": {}}, + "spec": {}, + "status": { + "conditions": [{"type": "Ready", "status": "True"}] if ready else [], + "serviceFQDN": FQDN, + }, + } + k8s.pods[pod_name] = {"metadata": {"name": pod_name, "uid": POD_UID}} + k8s.logs[pod_name] = ( + "booting\n" + f"dsh web: http://10.0.0.7:3080/?token={TOKEN}-launch\n" + ) + return sandbox_name, pod_name @pytest.fixture -def gw(monkeypatch): - """A freshly loaded gateway module (own last_activity/in_flight state) - with its Kubernetes CustomObjectsApi mocked, plus a Flask test client.""" - mock_crd = mock.MagicMock() - module = _load_gateway(monkeypatch, mock_crd) - module.app.testing = True # propagate unhandled exceptions instead of a bare 500 - return SimpleNamespace(module=module, crd=mock_crd, client=module.app.test_client()) +def env(): + settings = Settings( + namespace="test-ns", + pool="test-pool", + idle_timeout=60, + wake_timeout=1, + sweep_interval=3600, + upstream_retry_delay=0, + ) + k8s = FakeK8s() + upstream = FakeUpstream() + http_client = httpx.AsyncClient( + transport=httpx.MockTransport(upstream), timeout=5 + ) + ws_connect = FakeWSConnect() + app = create_app( + settings=settings, + k8s=k8s, + http_client=http_client, + ws_connect=ws_connect, + ) + return SimpleNamespace( + app=app, + gateway=app.state.gateway, + settings=settings, + k8s=k8s, + upstream=upstream, + ws_connect=ws_connect, + http_client=http_client, + ) -def crd_get_side_effect(claims=None, sandboxes=None): - """Builds a get_namespaced_custom_object side_effect that serves - "sandboxclaims" from `claims` and "sandboxes" from `sandboxes` (both - {name: obj_or_None} maps), raising a 404 ApiException for anything - missing -- mirroring how get_claim()/get_sandbox() interpret 404.""" - claims = claims or {} - sandboxes = sandboxes or {} - - def _get(group, version, namespace, plural, name): - if plural == "sandboxclaims": - table = claims - elif plural == "sandboxes": - table = sandboxes - else: - raise AssertionError(f"unexpected plural {plural!r}") - obj = table.get(name) - if obj is None: - raise k8s_client.ApiException(status=404) - return obj - - return _get +def asgi_client(env, cookies=None): + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=env.app), + base_url="http://gateway.test", + ) + if cookies: + client.cookies.update(cookies) + return client -# --- condition() --------------------------------------------------------- +def stale_handshake(env): + env.gateway.state.handshake[("alice", POD_UID, "gateway.test")] = ( + "dsh-auth-a=stale" + ) -def test_condition_true_when_status_matches(gw): + +# --- pure helpers ---------------------------------------------------------- + + +def test_condition_true_when_status_matches(): obj = {"status": {"conditions": [{"type": "Ready", "status": "True"}]}} - assert gw.module.condition(obj, "Ready") is True + assert condition(obj, "Ready") is True -def test_condition_false_when_status_is_false(gw): +def test_condition_false_when_status_is_false(): obj = {"status": {"conditions": [{"type": "Ready", "status": "False"}]}} - assert gw.module.condition(obj, "Ready") is False + assert condition(obj, "Ready") is False -def test_condition_false_when_type_absent(gw): +def test_condition_false_when_type_absent(): obj = {"status": {"conditions": [{"type": "Suspended", "status": "True"}]}} - assert gw.module.condition(obj, "Ready") is False + assert condition(obj, "Ready") is False -def test_condition_false_when_no_status(gw): - assert gw.module.condition({}, "Ready") is False +def test_condition_false_when_no_status(): + assert condition({}, "Ready") is False -# --- derive_state() -------------------------------------------------------- - -def test_derive_state_provisioning_when_sandbox_missing(gw): - assert gw.module.derive_state(None) == "Provisioning" +def test_derive_state_provisioning_when_sandbox_missing(): + assert derive_state(None) == "Provisioning" -def test_derive_state_ready(gw): +def test_derive_state_ready(): sandbox = {"spec": {}, "status": {"conditions": [{"type": "Ready", "status": "True"}]}} - assert gw.module.derive_state(sandbox) == "Ready" + assert derive_state(sandbox) == "Ready" -def test_derive_state_waking_when_not_yet_ready(gw): +def test_derive_state_waking_when_not_yet_ready(): sandbox = {"spec": {"operatingMode": "Running"}, "status": {"conditions": []}} - assert gw.module.derive_state(sandbox) == "Waking" + assert derive_state(sandbox) == "Waking" -def test_derive_state_suspended(gw): - sandbox = {"spec": {"operatingMode": "Suspended"}, - "status": {"conditions": [{"type": "Suspended", "status": "True"}]}} - assert gw.module.derive_state(sandbox) == "Suspended" +def test_derive_state_suspended(): + sandbox = { + "spec": {"operatingMode": "Suspended"}, + "status": {"conditions": [{"type": "Suspended", "status": "True"}]}, + } + assert derive_state(sandbox) == "Suspended" -def test_derive_state_suspending_while_mode_flipped_but_not_settled(gw): +def test_derive_state_suspending_while_mode_flipped_but_not_settled(): sandbox = {"spec": {"operatingMode": "Suspended"}, "status": {"conditions": []}} - assert gw.module.derive_state(sandbox) == "Suspending" + assert derive_state(sandbox) == "Suspending" -# --- authorized() ---------------------------------------------------------- - -def _claim_for_token(module, token): - return {"metadata": {"annotations": { - module.TOKEN_ANNOTATION: hashlib.sha256(token.encode()).hexdigest()}}} +def test_claim_name_and_user_roundtrip(): + assert claim_name("alice") == "dsh-alice" + assert claim_user({"metadata": {"name": "dsh-alice"}}) == "alice" -def test_authorized_accepts_correct_bearer_token(gw): - claim = _claim_for_token(gw.module, "s3cret") - with gw.module.app.test_request_context(headers={"Authorization": "Bearer s3cret"}): - assert gw.module.authorized(claim) is True +def test_parse_dsh_launch_token_from_real_log_line(): + log = ( + "2026-10-21T10:00:00Z starting\n" + "dsh web: http://10.244.1.7:3080/?token=abc-_123XYZ\n" + "some later line\n" + ) + assert parse_dsh_launch_token(log) == "abc-_123XYZ" -def test_authorized_rejects_wrong_token(gw): - claim = _claim_for_token(gw.module, "s3cret") - with gw.module.app.test_request_context(headers={"Authorization": "Bearer wrong"}): - assert gw.module.authorized(claim) is False +def test_parse_dsh_launch_token_uses_last_match(): + log = ( + "dsh web: http://10.0.0.1:3080/?token=old\n" + "restart\n" + "dsh web: http://10.0.0.2:3080/?token=new\n" + ) + assert parse_dsh_launch_token(log) == "new" -def test_authorized_rejects_missing_header(gw): - claim = _claim_for_token(gw.module, "s3cret") - with gw.module.app.test_request_context(): - assert gw.module.authorized(claim) is False +@pytest.mark.parametrize( + "log", + ["", "no token here", "dsh web: http://10.0.0.1:3080/?token=", "dsh web: garbage"], +) +def test_parse_dsh_launch_token_returns_none_without_token(log): + assert parse_dsh_launch_token(log) is None -def test_authorized_rejects_empty_bearer_token(gw): - claim = _claim_for_token(gw.module, "") - with gw.module.app.test_request_context(headers={"Authorization": "Bearer "}): - # bool(token) guards against an empty token matching an empty/unset - # annotation hash. - assert gw.module.authorized(claim) is False +def test_extract_dsh_cookies_keeps_only_dsh_auth_pairs(): + headers = [ + "dsh-auth-a=one; Path=/; HttpOnly; SameSite=Strict", + "other=two; Path=/", + "dsh-auth-b=three; Path=/", + ] + assert extract_dsh_cookies(headers) == "dsh-auth-a=one; dsh-auth-b=three" -# --- POST /users (create_user) --------------------------------------------- +def test_same_origin_origin_header(): + assert same_origin({"host": "gw.test", "origin": "http://gw.test"}) is True + assert same_origin({"host": "gw.test", "origin": "http://evil.test"}) is False -@pytest.mark.parametrize("bad_user", ["Alice", "-alice", "alice-", "", "a_b", 123, None]) -def test_create_user_rejects_invalid_dns1123_label(gw, bad_user): - response = gw.client.post("/users", json={"user": bad_user}) + +def test_same_origin_referer_fallback_and_missing(): + assert same_origin({"host": "gw.test", "referer": "http://gw.test/u/a/"}) is True + assert same_origin({"host": "gw.test"}) is False + + +def test_rewrite_location_prefixes_absolute_paths_only(): + assert rewrite_location("/", "alice") == "/u/alice/" + assert rewrite_location("/assets/app.js", "alice") == "/u/alice/assets/app.js" + assert rewrite_location("http://x/absolute", "alice") == "http://x/absolute" + assert rewrite_location("//other/path", "alice") == "//other/path" + assert rewrite_location("/u/alice/already", "alice") == "/u/alice/already" + + +def test_rewrite_location_drops_launch_token(): + assert rewrite_location("/?token=secret", "alice") == "/u/alice/" + + +def test_filtered_query_removes_token_only(): + assert filtered_query("a=1&token=x&b=2&token=y") == "a=1&b=2" + + +def test_token_matches_is_timing_safe_hash_check(): + claim = { + "metadata": { + "annotations": { + "aaas.example.com/token-sha256": hashlib.sha256( + TOKEN.encode() + ).hexdigest() + } + } + } + assert token_matches(claim, TOKEN) is True + assert token_matches(claim, "wrong") is False + assert token_matches(claim, "") is False + assert token_matches({"metadata": {}}, TOKEN) is False + + +def test_settings_from_env(monkeypatch): + monkeypatch.setenv("NAMESPACE", "ns-x") + monkeypatch.setenv("POOL", "pool-x") + monkeypatch.setenv("IDLE_TIMEOUT", "7") + monkeypatch.setenv("WAKE_TIMEOUT", "8") + monkeypatch.setenv("DSH_PORT", "9999") + monkeypatch.setenv("DSH_LOG_TAIL_LINES", "42") + monkeypatch.setenv("COOKIE_NAME", "sid") + monkeypatch.setenv("COOKIE_SECURE", "true") + monkeypatch.setenv("COOKIE_MAX_AGE", "100") + monkeypatch.setenv("MAX_BODY_MB", "2") + monkeypatch.setenv("HTTP_TIMEOUT", "12.5") + settings = Settings.from_env() + assert settings.namespace == "ns-x" + assert settings.pool == "pool-x" + assert settings.idle_timeout == 7 + assert settings.wake_timeout == 8 + assert settings.dsh_port == 9999 + assert settings.dsh_log_tail_lines == 42 + assert settings.cookie_name == "sid" + assert settings.cookie_secure is True + assert settings.cookie_max_age == 100 + assert settings.max_body_bytes == 2 * 1024 * 1024 + assert settings.http_timeout == 12.5 + + +# --- Kubernetes adapter ---------------------------------------------------- + + +@pytest.mark.asyncio +async def test_kubernetes_create_claim_body(monkeypatch): + from gateway import Kubernetes + + calls = [] + + async def fake_call(which, method, *args): + calls.append((which, method, args)) + + k8s = Kubernetes("test-ns") + monkeypatch.setattr(k8s, "_call", fake_call) + await k8s.create_claim("alice", "hash", "test-pool") + + which, method, args = calls[0] + assert which == "crd" + assert method == "create_namespaced_custom_object" + body = args[4] + assert body["metadata"]["name"] == "dsh-alice" + assert body["metadata"]["annotations"]["aaas.example.com/token-sha256"] == "hash" + assert body["spec"]["warmPoolRef"]["name"] == "test-pool" + assert body["spec"]["additionalPodMetadata"]["labels"][ + "sandbox.users.io/dsh-user" + ] == "alice" + + +@pytest.mark.asyncio +async def test_kubernetes_maps_404_to_none_and_propagates_other_errors(monkeypatch): + from gateway import Kubernetes + + k8s = Kubernetes("test-ns") + calls = {"n": 0} + + async def fake_call(which, method, *args): + calls["n"] += 1 + raise ApiException(status=404 if calls["n"] == 1 else 500) + + monkeypatch.setattr(k8s, "_call", fake_call) + assert await k8s.get_claim("alice") is None + with pytest.raises(ApiException): + await k8s.get_claim("alice") + + +@pytest.mark.asyncio +async def test_kubernetes_read_pod_logs_forwards_container_and_tail(monkeypatch): + from gateway import Kubernetes + + calls = [] + + class Core: + def read_namespaced_pod_log(self, *args, **kwargs): + calls.append((args, kwargs)) + return "log line" + + k8s = Kubernetes("test-ns") + monkeypatch.setattr(k8s, "_clients", lambda: (None, Core())) + assert await k8s.read_pod_logs("pod-1", "dsh-web", 200) == "log line" + assert calls == [(("pod-1", "test-ns"), {"container": "dsh-web", "tail_lines": 200})] + + +# --- health / landing / redirect ------------------------------------------ + + +@pytest.mark.asyncio +async def test_healthz_needs_no_auth_and_no_cluster(env): + async with asgi_client(env) as client: + response = await client.get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.asyncio +async def test_healthz_does_not_touch_the_cluster_at_startup(): + # No k8s injected: create_app builds the lazy Kubernetes adapter, which + # must not load credentials until a claim/pod call actually happens. + app = create_app( + settings=Settings(namespace="test-ns"), + http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda r: dsh_response())), + ws_connect=lambda *a, **k: None, + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://gateway.test", + ) as client: + response = await client.get("/healthz") + assert response.status_code == 200 + + +@pytest.mark.asyncio +async def test_landing_page_has_signup_and_login_forms(env): + async with asgi_client(env) as client: + response = await client.get("/") + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + assert 'action="/users"' in response.text + assert 'action="/login"' in response.text + + +@pytest.mark.asyncio +async def test_landing_page_shows_session_when_cookie_present(env): + async with asgi_client( + env, cookies={"dsh_gw_session": f"alice:{TOKEN}"} + ) as client: + response = await client.get("/") + assert "Signed in as" in response.text + assert "/u/alice/" in response.text + + +@pytest.mark.asyncio +async def test_workspace_redirect_adds_trailing_slash(env): + async with asgi_client(env) as client: + response = await client.get("/u/alice?x=1") + assert response.status_code == 308 + assert response.headers["location"] == "/u/alice/?x=1" + + +# --- signup / login / logout ---------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_user", ["Alice", "-alice", "alice-", "", "a_b", 123, None, "a" * 60] +) +async def test_signup_rejects_invalid_dns1123_label(env, bad_user): + async with asgi_client(env) as client: + response = await client.post("/users", json={"user": bad_user}) assert response.status_code == 400 - gw.crd.create_namespaced_custom_object.assert_not_called() + assert env.k8s.created == [] -def test_create_user_rejects_label_over_max_len(gw): - too_long = "a" * (gw.module.MAX_USER_LEN + 1) - response = gw.client.post("/users", json={"user": too_long}) - assert response.status_code == 400 - gw.crd.create_namespaced_custom_object.assert_not_called() - - -def test_create_user_conflict_when_claim_already_exists(gw): - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": {"metadata": {"name": "dsh-alice"}}}) - - response = gw.client.post("/users", json={"user": "alice"}) - +@pytest.mark.asyncio +async def test_signup_conflict_when_claim_exists(env): + add_user(env.k8s) + async with asgi_client(env) as client: + response = await client.post("/users", json={"user": "alice"}) assert response.status_code == 409 - gw.crd.create_namespaced_custom_object.assert_not_called() + assert env.k8s.created == [] -def test_create_user_conflict_on_concurrent_signup_race(gw): - gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) - gw.crd.create_namespaced_custom_object.side_effect = k8s_client.ApiException(status=409) - - response = gw.client.post("/users", json={"user": "alice"}) - +@pytest.mark.asyncio +async def test_signup_conflict_on_concurrent_race(env): + env.k8s.conflict_on_create = True + async with asgi_client(env) as client: + response = await client.post("/users", json={"user": "alice"}) assert response.status_code == 409 -def test_create_user_success_stores_only_token_hash(gw, monkeypatch): - gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) - ready_sandbox = { +@pytest.mark.asyncio +async def test_signup_json_stores_only_hash_and_returns_token(env, monkeypatch): + sandbox = { "metadata": {"name": "dsh-alice-sbx"}, "spec": {}, "status": {"conditions": [{"type": "Ready", "status": "True"}]}, } - # wait_ready() polls with time.sleep(1); stub it out rather than racing - # a real poll loop in a unit test. - monkeypatch.setattr(gw.module, "wait_ready", lambda user, timeout: ready_sandbox) - response = gw.client.post("/users", json={"user": "alice"}) + async def ready(user, timeout): + return sandbox + + monkeypatch.setattr(env.gateway, "wait_ready", ready) + async with asgi_client(env) as client: + response = await client.post("/users", json={"user": "alice"}) assert response.status_code == 201 - body = response.get_json() + body = response.json() assert body["user"] == "alice" assert body["state"] == "Ready" assert body["sandbox"] == "dsh-alice-sbx" - assert body["token"] # shown once in the response... - - gw.crd.create_namespaced_custom_object.assert_called_once() - claim_body = gw.crd.create_namespaced_custom_object.call_args.args[4] - assert claim_body["metadata"]["name"] == "dsh-alice" - # ...but only its SHA-256 is ever sent to the API server. - want_hash = hashlib.sha256(body["token"].encode()).hexdigest() - assert claim_body["metadata"]["annotations"][gw.module.TOKEN_ANNOTATION] == want_hash - assert body["token"] not in json.dumps(claim_body) - assert claim_body["spec"]["warmPoolRef"]["name"] == "test-pool" - assert claim_body["spec"]["additionalPodMetadata"]["labels"]["sandbox.users.io/dsh-user"] == "alice" + assert body["token"] + user, token_hash, pool = env.k8s.created[0] + assert user == "alice" + assert pool == "test-pool" + assert token_hash == hashlib.sha256(body["token"].encode()).hexdigest() + assert token_hash != body["token"] + # The JSON API also gets a browser session cookie (HttpOnly, SameSite=Lax). + cookie = response.headers["set-cookie"] + assert "dsh_gw_session=alice:" in cookie + assert "HttpOnly" in cookie + assert "SameSite=lax" in cookie -# --- GET /users/ and DELETE /users/ ----------------------------- +@pytest.mark.asyncio +async def test_signup_returns_202_when_sandbox_not_ready(env, monkeypatch): + async def not_ready(user, timeout): + return None -def test_get_user_not_found(gw): - gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) - response = gw.client.get("/users/alice") + monkeypatch.setattr(env.gateway, "wait_ready", not_ready) + async with asgi_client(env) as client: + response = await client.post("/users", json={"user": "alice"}) + assert response.status_code == 202 + assert response.json()["state"] == "Provisioning" + + +@pytest.mark.asyncio +async def test_signup_form_redirects_and_sets_session_cookie(env, monkeypatch): + async def not_ready(user, timeout): + return None + + monkeypatch.setattr(env.gateway, "wait_ready", not_ready) + async with asgi_client(env) as client: + response = await client.post( + "/users", + data={"user": "bob"}, + headers={"content-type": "application/x-www-form-urlencoded"}, + ) + assert response.status_code == 303 + assert response.headers["location"] == "/u/bob/" + cookie = response.headers["set-cookie"] + assert re_cookie_user(cookie) == "bob" + + +def re_cookie_user(set_cookie: str) -> str: + value = set_cookie.split(";", 1)[0].split("=", 1)[1] + return value.split(":", 1)[0] + + +@pytest.mark.asyncio +async def test_signup_body_limit_returns_413(env): + env.settings = Settings(**{**env.settings.__dict__, "max_body_bytes": 8}) + env.gateway.settings = env.settings + async with asgi_client(env) as client: + response = await client.post( + "/users", + content=json.dumps({"user": "alice"}), + headers={"content-type": "application/json"}, + ) + assert response.status_code == 413 + + +@pytest.mark.asyncio +async def test_login_form_sets_cookie_and_redirects(env): + add_user(env.k8s) + async with asgi_client(env) as client: + response = await client.post( + "/login", + data={"user": "alice", "token": TOKEN}, + headers={"content-type": "application/x-www-form-urlencoded"}, + ) + assert response.status_code == 303 + assert response.headers["location"] == "/u/alice/" + assert re_cookie_user(response.headers["set-cookie"]) == "alice" + + +@pytest.mark.asyncio +async def test_login_json_and_failures(env): + add_user(env.k8s) + async with asgi_client(env) as client: + ok = await client.post("/login", json={"user": "alice", "token": TOKEN}) + bad = await client.post("/login", json={"user": "alice", "token": "nope"}) + missing = await client.post("/login", json={"user": "ghost", "token": TOKEN}) + assert ok.status_code == 200 + assert bad.status_code == 401 + assert missing.status_code == 404 + + +@pytest.mark.asyncio +async def test_logout_clears_cookie(env): + async with asgi_client( + env, cookies={"dsh_gw_session": f"alice:{TOKEN}"} + ) as client: + response = await client.post("/logout") + assert response.status_code == 303 + assert response.headers["location"] == "/" + assert "dsh_gw_session=" in response.headers["set-cookie"] + assert "Max-Age=0" in response.headers["set-cookie"] + + +# --- auth ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_get_user_requires_valid_bearer_or_cookie(env): + add_user(env.k8s) + async with asgi_client(env) as client: + forbidden = await client.get( + "/users/alice", headers={"Authorization": "Bearer wrong"} + ) + bearer = await client.get( + "/users/alice", headers={"Authorization": f"Bearer {TOKEN}"} + ) + cookie = await client.get( + "/users/alice", + headers={"Cookie": f"dsh_gw_session=alice:{TOKEN}"}, + ) + wrong_user_cookie = await client.get( + "/users/alice", + headers={"Cookie": f"dsh_gw_session=bob:{TOKEN}"}, + ) + assert forbidden.status_code == 401 + assert bearer.status_code == 200 + assert bearer.json()["state"] == "Ready" + assert cookie.status_code == 200 + assert wrong_user_cookie.status_code == 401 + + +@pytest.mark.asyncio +async def test_get_user_not_found(env): + async with asgi_client(env) as client: + response = await client.get("/users/alice") assert response.status_code == 404 -def test_get_user_requires_valid_token(gw): - claim = _claim_for_token(gw.module, "s3cret") - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}) +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/users/Alice", "/u/Alice/", "/u/Alice"]) +async def test_invalid_user_in_path_is_404(env, path): + async with asgi_client(env) as client: + response = await client.get(path) + assert response.status_code == 404 + assert env.k8s.claims == {} - response = gw.client.get("/users/alice", headers={"Authorization": "Bearer wrong"}) +@pytest.mark.asyncio +async def test_delete_user_requires_auth(env): + add_user(env.k8s) + async with asgi_client(env) as client: + response = await client.delete( + "/users/alice", headers={"Authorization": "Bearer wrong"} + ) assert response.status_code == 401 + assert env.k8s.deleted == [] -def test_get_user_returns_state_when_authorized(gw): - claim = _claim_for_token(gw.module, "s3cret") - sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, - "status": {"conditions": [{"type": "Ready", "status": "True"}]}} - claim["status"] = {"sandbox": {"name": "dsh-alice-sbx"}} - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}, sandboxes={"dsh-alice-sbx": sandbox}) +@pytest.mark.asyncio +async def test_delete_with_cookie_requires_same_origin(env): + add_user(env.k8s) + env.gateway.state.handshake[("alice", POD_UID, "gateway.test")] = "dsh-auth-a=x" + cookies = {"dsh_gw_session": f"alice:{TOKEN}"} + async with asgi_client(env, cookies=cookies) as client: + missing = await client.delete("/users/alice") + cross = await client.delete( + "/users/alice", headers={"Origin": "http://evil.test"} + ) + good = await client.delete( + "/users/alice", headers={"Origin": "http://gateway.test"} + ) + assert missing.status_code == 403 + assert cross.status_code == 403 + assert good.status_code == 200 + assert env.k8s.deleted == ["alice"] + assert env.gateway.state.handshake == {} - response = gw.client.get("/users/alice", headers={"Authorization": "Bearer s3cret"}) +@pytest.mark.asyncio +async def test_delete_with_bearer_does_not_need_origin(env): + add_user(env.k8s) + async with asgi_client(env) as client: + response = await client.delete( + "/users/alice", headers={"Authorization": f"Bearer {TOKEN}"} + ) assert response.status_code == 200 - assert response.get_json()["state"] == "Ready" -def test_delete_user_not_found(gw): - gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) - response = gw.client.delete("/users/alice") +@pytest.mark.asyncio +async def test_delete_missing_claim_is_404(env): + add_user(env.k8s) + env.k8s.claims.clear() + async with asgi_client(env) as client: + response = await client.delete( + "/users/alice", headers={"Authorization": f"Bearer {TOKEN}"} + ) assert response.status_code == 404 -def test_delete_user_requires_valid_token(gw): - claim = _claim_for_token(gw.module, "s3cret") - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}) +@pytest.mark.asyncio +async def test_unexpected_api_error_propagates(env): + add_user(env.k8s) + env.k8s.get_claim_error = ApiException(status=500) + async with asgi_client(env) as client: + with pytest.raises(ApiException): + await client.get("/users/alice") - response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer wrong"}) +# --- DSH handshake --------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_proxy_handshakes_and_uses_browser_host(env): + add_user(env.k8s) + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 200 + assert response.text == "hello from DSH" + handshake = env.upstream.handshake_requests[0] + assert handshake.url.host == FQDN + assert handshake.url.port == 3080 + assert handshake.url.params["token"] == TOKEN + "-launch" + assert handshake.headers["host"] == "gateway.test" + proxy = env.upstream.proxy_requests[0] + assert proxy.headers["cookie"] == "dsh-auth-a=one" + + +@pytest.mark.asyncio +async def test_dsh_cookie_is_cached_per_pod_uid_and_host(env): + add_user(env.k8s) + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + first = await client.get("/u/alice/api/one") + second = await client.get("/u/alice/api/two") + assert first.status_code == second.status_code == 200 + assert len(env.upstream.handshake_requests) == 1 + assert len(env.upstream.proxy_requests) == 2 + + +@pytest.mark.asyncio +async def test_missing_launch_token_returns_503_with_retry_after(env): + _sandbox_name, pod_name = add_user(env.k8s) + env.k8s.logs[pod_name] = "no token printed yet\n" + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 503 + assert response.headers["retry-after"] == "10" + assert "launch token" in response.json()["error"] + + +@pytest.mark.asyncio +async def test_handshake_4xx_is_503(env): + add_user(env.k8s) + env.upstream.handshake_status = 401 + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 503 + + +@pytest.mark.asyncio +async def test_missing_cookie_in_handshake_is_503(env): + add_user(env.k8s) + env.upstream.handshake_cookies = ["other=1; Path=/"] + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 503 + + +@pytest.mark.asyncio +async def test_pod_annotation_overrides_sandbox_name(env): + sandbox_name, pod_name = add_user(env.k8s) + env.k8s.sandboxes[sandbox_name]["metadata"]["annotations"][ + "agents.x-k8s.io/pod-name" + ] = "custom-pod" + env.k8s.pods["custom-pod"] = {"metadata": {"uid": "other-uid"}} + env.k8s.logs["custom-pod"] = env.k8s.logs[pod_name] + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 200 + assert ("alice", "other-uid", "gateway.test") in env.gateway.state.handshake + + +# --- HTTP proxy ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_proxy_strips_gateway_auth_cookie_and_hop_headers(env): + add_user(env.k8s) + + def handler(request): + return dsh_response( + 200, + b"ok", + { + "content-type": "text/plain", + "connection": "keep-alive", + "set-cookie": "upstream=leak; Path=/", + }, + ) + + env.upstream.proxy_handler = handler + headers = { + "Authorization": f"Bearer {TOKEN}", + "Cookie": "dsh_gw_session=alice:x; browser=1", + "X-Forwarded-For": "1.2.3.4", + "X-Forwarded-Proto": "https", + "Host": "gateway.test", + "Origin": "http://gateway.test", + } + async with asgi_client(env) as client: + response = await client.get("/u/alice/api/foo?token=secret&x=1", headers=headers) + + assert response.status_code == 200 + proxy = env.upstream.proxy_requests[0] + assert proxy.url.path == "/api/foo" + assert proxy.url.query.decode() == "x=1" + assert "authorization" not in proxy.headers + assert proxy.headers["cookie"] == "dsh-auth-a=one" + assert "browser=1" not in proxy.headers["cookie"] + assert proxy.headers["host"] == "gateway.test" + assert proxy.headers["origin"] == "http://gateway.test" + assert proxy.headers["x-forwarded-for"] == "127.0.0.1" + assert proxy.headers["x-forwarded-proto"] == "http" + # Upstream Set-Cookie and hop-by-hop headers never reach the browser. + assert "set-cookie" not in response.headers + assert "connection" not in response.headers + + +@pytest.mark.asyncio +async def test_proxy_rewrites_absolute_path_location(env): + add_user(env.k8s) + env.upstream.proxy_handler = lambda request: dsh_response( + 303, + headers=[ + ("location", "/?token=leak&next=1"), + ("set-cookie", "upstream=leak; Path=/"), + ], + ) + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/redirect", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/u/alice/?next=1" + assert "set-cookie" not in response.headers + + +@pytest.mark.asyncio +async def test_proxy_refreshes_cookie_once_on_upstream_401(env): + add_user(env.k8s) + env.upstream.handshake_cookies = ["dsh-auth-a=new; Path=/"] + stale_handshake(env) + calls = [] + + def handler(request): + calls.append(request) + if len(calls) == 1: + return dsh_response(401, b"expired") + return dsh_response(200, b"fresh") + + env.upstream.proxy_handler = handler + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 200 + assert response.text == "fresh" + assert len(calls) == 2 + assert len(env.upstream.handshake_requests) == 1 # only the forced refresh + assert calls[1].headers["cookie"] == "dsh-auth-a=new" + + +@pytest.mark.asyncio +async def test_proxy_does_not_refresh_401_without_cached_cookie(env): + add_user(env.k8s) + env.upstream.proxy_handler = lambda request: dsh_response(401, b"no") + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") assert response.status_code == 401 - gw.crd.delete_namespaced_custom_object.assert_not_called() + assert len(env.upstream.proxy_requests) == 1 -def test_delete_user_cascades_and_clears_activity(gw): - claim = _claim_for_token(gw.module, "s3cret") - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}) - gw.module.last_activity["alice"] = time.time() +@pytest.mark.asyncio +async def test_proxy_upstream_connection_error_is_502(env): + add_user(env.k8s) - response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) + def handler(request): + raise httpx.ConnectError("boom", request=request) + env.upstream.proxy_handler = handler + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 502 + assert env.gateway.state.in_flight["alice"] == 0 + + +@pytest.mark.asyncio +async def test_proxy_body_limit_returns_413(env): + add_user(env.k8s) + env.gateway.settings = Settings( + **{**env.settings.__dict__, "max_body_bytes": 4} + ) + async with asgi_client(env) as client: + response = await client.post( + "/u/alice/api/echo", + content=b"too large", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + assert response.status_code == 413 + assert env.gateway.state.in_flight["alice"] == 0 + + +@pytest.mark.asyncio +async def test_proxy_wakes_suspended_sandbox(env, monkeypatch): + sandbox_name, _ = add_user(env.k8s, ready=False) + env.k8s.sandboxes[sandbox_name]["spec"]["operatingMode"] = "Suspended" + env.upstream.proxy_handler = lambda request: dsh_response(200, b"awake") + + async def ready(user, timeout): + env.k8s.sandboxes[sandbox_name]["spec"]["operatingMode"] = "Running" + env.k8s.sandboxes[sandbox_name]["status"] = { + "conditions": [{"type": "Ready", "status": "True"}], + "serviceFQDN": FQDN, + } + return env.k8s.sandboxes[sandbox_name] + + monkeypatch.setattr(env.gateway, "wait_ready", ready) + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") assert response.status_code == 200 - gw.crd.delete_namespaced_custom_object.assert_called_once() - assert "alice" not in gw.module.last_activity + assert ("dsh-alice-sbx", "Running") in env.k8s.patches -def test_delete_user_already_gone_is_idempotent(gw): - claim = _claim_for_token(gw.module, "s3cret") - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}) - gw.crd.delete_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) +@pytest.mark.asyncio +async def test_proxy_wake_timeout_returns_503(env, monkeypatch): + sandbox_name, _ = add_user(env.k8s, ready=False) + env.k8s.sandboxes[sandbox_name]["spec"]["operatingMode"] = "Suspended" - response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) + async def never(user, timeout): + return None - assert response.status_code == 200 + monkeypatch.setattr(env.gateway, "wait_ready", never) + async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 503 + assert response.headers["retry-after"] == "10" -def test_delete_user_unexpected_api_error_propagates(gw): - claim = _claim_for_token(gw.module, "s3cret") - gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( - claims={"dsh-alice": claim}) - gw.crd.delete_namespaced_custom_object.side_effect = k8s_client.ApiException(status=500) - - with pytest.raises(k8s_client.ApiException): - gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) +@pytest.mark.asyncio +async def test_proxy_unknown_user_is_404(env): + async with asgi_client(env) as client: + response = await client.get("/u/ghost/api/state") + assert response.status_code == 404 -# --- idle_sweeper() ---------------------------------------------------------- - -def _run_one_sweep(gw): - # time.sleep(15) runs before the try/except in idle_sweeper's loop body, - # so the first call must succeed (to let one pass execute) and only the - # second call breaks out. - with mock.patch("time.sleep", side_effect=[None, _StopLoop()]): - with pytest.raises(_StopLoop): - gw.module.idle_sweeper() +@pytest.mark.asyncio +async def test_proxy_requires_auth(env): + add_user(env.k8s) + async with asgi_client(env) as client: + response = await client.get("/u/alice/api/state") + assert response.status_code == 401 -def test_idle_sweeper_suspends_idle_ready_sandbox(gw): - sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, - "status": {"conditions": [{"type": "Ready", "status": "True"}]}} - claim_item = {"metadata": {"name": "dsh-alice"}, - "status": {"sandbox": {"name": "dsh-alice-sbx"}}} - gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} - gw.crd.get_namespaced_custom_object.return_value = sandbox - gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 - - _run_one_sweep(gw) - - gw.crd.patch_namespaced_custom_object.assert_called_once() - args = gw.crd.patch_namespaced_custom_object.call_args.args - assert args[4] == "dsh-alice-sbx" - assert args[5] == {"spec": {"operatingMode": "Suspended"}} +@pytest.mark.asyncio +async def test_proxy_cookie_mutation_requires_same_origin(env): + add_user(env.k8s) + cookies = {"dsh_gw_session": f"alice:{TOKEN}"} + async with asgi_client(env, cookies=cookies) as client: + cross = await client.post( + "/u/alice/api/mutate", headers={"Origin": "http://evil.test"}, content=b"{}" + ) + good = await client.post( + "/u/alice/api/mutate", + headers={"Origin": "http://gateway.test"}, + content=b"{}", + ) + assert cross.status_code == 403 + assert good.status_code == 200 -def test_idle_sweeper_skips_user_with_in_flight_request(gw): - sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, - "status": {"conditions": [{"type": "Ready", "status": "True"}]}} - claim_item = {"metadata": {"name": "dsh-alice"}, - "status": {"sandbox": {"name": "dsh-alice-sbx"}}} - gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} - gw.crd.get_namespaced_custom_object.return_value = sandbox - gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 - gw.module.in_flight["alice"] = 1 - - _run_one_sweep(gw) - - gw.crd.patch_namespaced_custom_object.assert_not_called() +# --- WebSocket proxy ------------------------------------------------------- -def test_idle_sweeper_skips_recently_active_user(gw): - sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, - "status": {"conditions": [{"type": "Ready", "status": "True"}]}} - claim_item = {"metadata": {"name": "dsh-alice"}, - "status": {"sandbox": {"name": "dsh-alice-sbx"}}} - gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} - gw.crd.get_namespaced_custom_object.return_value = sandbox - gw.module.last_activity["alice"] = time.time() - - _run_one_sweep(gw) - - gw.crd.patch_namespaced_custom_object.assert_not_called() +def test_ws_unauthenticated_closes_4401(env): + with TestClient(env.app) as client: + with client.websocket_connect("/u/alice/api/remote.mux") as ws: + with pytest.raises(WebSocketDisconnect) as exc: + ws.receive_text() + assert exc.value.code == 4401 -def test_idle_sweeper_skips_sandbox_not_ready(gw): - sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, "status": {"conditions": []}} - claim_item = {"metadata": {"name": "dsh-alice"}, - "status": {"sandbox": {"name": "dsh-alice-sbx"}}} - gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} - gw.crd.get_namespaced_custom_object.return_value = sandbox - gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 +def test_ws_cookie_cross_origin_closes_4401(env): + add_user(env.k8s) + with TestClient(env.app) as client: + with client.websocket_connect( + "/u/alice/api/remote.mux", + headers={ + "cookie": f"dsh_gw_session=alice:{TOKEN}", + "origin": "http://evil.test", + }, + ) as ws: + with pytest.raises(WebSocketDisconnect) as exc: + ws.receive_text() + assert exc.value.code == 4401 - _run_one_sweep(gw) - gw.crd.patch_namespaced_custom_object.assert_not_called() +def test_ws_bridges_frames_and_injects_cookie(env): + add_user(env.k8s) + with TestClient(env.app) as client: + with client.websocket_connect( + "/u/alice/api/remote.mux?token=secret&x=1", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as ws: + ws.send_text("hello") + assert ws.receive_text() == "welcome" + assert ws.receive_text() == "pong" + + url, kwargs = env.ws_connect.calls[0] + assert url.startswith(f"ws://{FQDN}:3080/api/remote.mux") + assert "token=secret" not in url + assert "x=1" in url + headers = kwargs["additional_headers"] + assert "authorization" not in headers + assert headers["cookie"] == "dsh-auth-a=one" + assert headers["host"] == "testserver" + assert env.ws_connect.instances[0].sent == ["hello"] + + +def test_ws_cookie_same_origin_bridges_frames(env): + add_user(env.k8s) + with TestClient(env.app) as client: + with client.websocket_connect( + "/u/alice/api/remote.mux", + headers={ + "cookie": f"dsh_gw_session=alice:{TOKEN}", + "origin": "http://testserver", + }, + ) as ws: + ws.send_text("hello") + assert ws.receive_text() == "welcome" + assert ws.receive_text() == "pong" + + headers = env.ws_connect.calls[0][1]["additional_headers"] + assert headers["cookie"] == "dsh-auth-a=one" + assert headers["origin"] == "http://testserver" + + +def test_ws_handshake_failure_closes_1011(env): + _, pod_name = add_user(env.k8s) + env.k8s.logs[pod_name] = "token not ready\n" + with TestClient(env.app) as client: + with client.websocket_connect( + "/u/alice/api/remote.mux", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as ws: + with pytest.raises(WebSocketDisconnect) as exc: + ws.receive_text() + assert exc.value.code == 1011 + + +def test_ws_upstream_connect_failure_closes_1011(env): + add_user(env.k8s) + env.ws_connect.error = RuntimeError("cannot connect") + with TestClient(env.app) as client: + with client.websocket_connect( + "/u/alice/api/remote.mux", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as ws: + with pytest.raises(WebSocketDisconnect) as exc: + ws.receive_text() + assert exc.value.code == 1011 + + +# --- idle sweeper ---------------------------------------------------------- + + +def ready_claim(user="alice"): + return { + "metadata": {"name": claim_name(user)}, + "status": {"sandbox": {"name": f"dsh-{user}-sbx"}}, + } + + +def ready_sandbox(user="alice"): + return { + "metadata": {"name": f"dsh-{user}-sbx"}, + "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + } + + +@pytest.mark.asyncio +async def test_sweeper_suspends_idle_ready_sandbox(env): + env.k8s.claims["alice"] = ready_claim() + env.k8s.sandboxes["dsh-alice-sbx"] = ready_sandbox() + env.gateway.state.last_activity["alice"] = ( + time.time() - env.settings.idle_timeout - 10 + ) + env.gateway.state.handshake[("alice", POD_UID, "gateway.test")] = "dsh-auth-a=x" + + await env.gateway.sweep_once() + + assert env.k8s.patches == [("dsh-alice-sbx", "Suspended")] + assert env.gateway.state.handshake == {} + assert env.gateway.state.maintenance["alice"] is False + + +@pytest.mark.asyncio +async def test_sweeper_skips_in_flight_request(env): + env.k8s.claims["alice"] = ready_claim() + env.k8s.sandboxes["dsh-alice-sbx"] = ready_sandbox() + env.gateway.state.last_activity["alice"] = time.time() - 1000 + env.gateway.state.in_flight["alice"] = 1 + + await env.gateway.sweep_once() + + assert env.k8s.patches == [] + + +@pytest.mark.asyncio +async def test_sweeper_skips_recently_active_user(env): + env.k8s.claims["alice"] = ready_claim() + env.k8s.sandboxes["dsh-alice-sbx"] = ready_sandbox() + env.gateway.state.last_activity["alice"] = time.time() + + await env.gateway.sweep_once() + + assert env.k8s.patches == [] + + +@pytest.mark.asyncio +async def test_sweeper_skips_sandbox_not_ready(env): + env.k8s.claims["alice"] = ready_claim() + env.k8s.sandboxes["dsh-alice-sbx"] = { + "metadata": {"name": "dsh-alice-sbx"}, + "spec": {}, + "status": {"conditions": []}, + } + env.gateway.state.last_activity["alice"] = time.time() - 1000 + + await env.gateway.sweep_once() + + assert env.k8s.patches == [] + + +@pytest.mark.asyncio +async def test_sweeper_survives_transient_api_error(env): + env.k8s.claims["alice"] = ready_claim() + env.k8s.sandboxes["dsh-alice-sbx"] = ready_sandbox() + env.gateway.state.last_activity["alice"] = time.time() - 1000 + env.k8s.patch_error = ApiException(status=500) + + with pytest.raises(ApiException): + await env.gateway.sweep_once() + + # The per-user maintenance flag must be released even on failure. + assert env.gateway.state.maintenance["alice"] is False diff --git a/examples/dsh-agents-as-a-service/run-test-kind.sh b/examples/dsh-agents-as-a-service/run-test-kind.sh index d502f87..457e839 100755 --- a/examples/dsh-agents-as-a-service/run-test-kind.sh +++ b/examples/dsh-agents-as-a-service/run-test-kind.sh @@ -12,7 +12,11 @@ cd "$(dirname "$0")" NS=dsh-demo pass() { echo "PASS: $1"; } fail() { echo "FAIL: $1" >&2; exit 1; } -cleanup() { kubectl delete namespace "$NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true; } +cleanup() { + kill "${PF:-}" 2>/dev/null || true + rm -f "${JAR:-}" + kubectl delete namespace "$NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} trap cleanup EXIT echo "=== 1. install prereqs (throwaway key) + template + warm pool" @@ -77,34 +81,146 @@ BOUND=$(kubectl -n "$NS" get sandboxclaim dsh-mallory -o jsonpath='{.status.sand kubectl delete -f 40-claim-rejected.yaml >/dev/null pass "claim rejected with EnvVarsInjectionRejected and never bound" -echo "=== 7. gateway: signup, proxy, idle-suspend, wake-on-connect" +echo "=== 7. gateway: browser signup, proxy, idle-suspend, wake-on-connect" if command -v docker >/dev/null && command -v kind >/dev/null; then KIND_CLUSTER="${KIND_CLUSTER:-$(kubectl config current-context | sed 's/^kind-//')}" - docker build -q -t dsh-aaas-gateway:demo gateway/ >/dev/null - kind load docker-image dsh-aaas-gateway:demo --name "$KIND_CLUSTER" >/dev/null + # Tag exactly what 50-gateway.yaml references so the smoke test exercises + # the shipped manifest unchanged (imagePullPolicy: IfNotPresent uses the + # image loaded into kind). + GW_IMAGE=ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway + docker build -q -t "$GW_IMAGE" gateway/ >/dev/null + kind load docker-image "$GW_IMAGE" --name "$KIND_CLUSTER" >/dev/null kubectl apply -f 50-gateway.yaml kubectl -n "$NS" rollout status deploy/dsh-gateway --timeout=120s # Local port-forward uses 18080 (NOT 3080, which stays in-cluster). kubectl -n "$NS" port-forward svc/dsh-gateway 18080:8080 >/dev/null 2>&1 & PF=$! - sleep 3 GW=localhost:18080 + JAR=$(mktemp) - SIGNUP=$(curl -s -X POST "http://$GW/users" -H 'Content-Type: application/json' -d '{"user":"bob"}') - TOKEN=$(echo "$SIGNUP" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p') - [ -n "$TOKEN" ] || fail "signup returned no token: $SIGNUP" - echo "$SIGNUP" | grep -q '"state":"Ready"' || fail "signup not Ready: $SIGNUP" - pass "gateway signup: bob Ready with one-time token" + for _ in $(seq 1 20); do + HEALTH=$(curl -s -o /dev/null -w '%{http_code}' "http://$GW/healthz" || true) + [ "$HEALTH" = "200" ] && break + sleep 1 + done + [ "${HEALTH:-}" = "200" ] || fail "gateway /healthz was not reachable on $GW" + pass "gateway health: /healthz -> 200" - curl -sf -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \ - || fail "proxied web call failed" - curl -s -o /dev/null -w '%{http_code}' "http://$GW/u/bob/" | grep -q 401 \ - || fail "tokenless request was not rejected" - pass "gateway proxy: authed request 200, tokenless 401" + # Signup exactly as the landing page does it: a form POST, answered with a + # session cookie. The browser never handles the DSH launch token. + SIGNUP_CODE=$(curl -s -o /tmp/dsh-gw-signup.out -w '%{http_code}' \ + -c "$JAR" -H "Origin: http://$GW" \ + -X POST "http://$GW/users" -d 'user=bob') + case "$SIGNUP_CODE" in + 201|202|303) ;; + *) fail "signup returned HTTP $SIGNUP_CODE: $(cat /tmp/dsh-gw-signup.out)" ;; + esac + grep -q 'dsh_gw_session' "$JAR" || fail "signup did not set the dsh_gw_session cookie" + STATE="" + for _ in $(seq 1 30); do + STATE=$(curl -s -b "$JAR" "http://$GW/users/bob" \ + | sed -n 's/.*"state":"\([^"]*\)".*/\1/p') + [ "$STATE" = "Ready" ] && break + sleep 2 + done + [ "$STATE" = "Ready" ] || fail "bob's claim was not Ready (state=$STATE)" + pass "gateway signup: bob Ready with an HttpOnly session cookie" + + CT=$(curl -s -o /tmp/dsh-u-bob.html -w '%{content_type}' -b "$JAR" "http://$GW/u/bob/") + case "$CT" in text/html*) ;; *) fail "/u/bob/ returned content-type '$CT', expected text/html";; esac + grep -qi '/dev/null 2>&1; then + python3 - "$JAR" "$GW" <<'PY' || fail "WebSocket checks failed" +import sys + +from websockets.exceptions import ConnectionClosed, InvalidStatus +from websockets.sync.client import connect + +jar_path, gw = sys.argv[1], sys.argv[2] +cookie = None +with open(jar_path, encoding="utf-8") as jar: + for raw in jar: + line = raw.removeprefix("#HttpOnly_") + if not line.strip() or line.startswith("#"): + continue + fields = line.rstrip("\n").split("\t") + if len(fields) >= 7 and fields[5] == "dsh_gw_session": + cookie = f"{fields[5]}={fields[6]}" + break +if not cookie: + sys.exit("FAIL: no dsh_gw_session cookie in the jar") + +uri = f"ws://{gw}/u/bob/api/remote.mux" + + +def connect_ws(headers): + try: + return connect(uri, additional_headers=headers, open_timeout=30) + except TypeError: # websockets < 14 + return connect(uri, extra_headers=headers, open_timeout=30) + + +def status_of(exc): + response = getattr(exc, "response", None) + return getattr(response, "status_code", None) or getattr(exc, "status_code", None) + + +# 1. Without the gateway cookie the gateway accepts the upgrade and then +# closes with application code 4401 (WebSocket auth has no HTTP 401 path). +try: + with connect(uri, open_timeout=10) as ws: + try: + ws.recv(timeout=10) + except ConnectionClosed as exc: + code = exc.rcvd.code if exc.rcvd else None + if code != 4401: + sys.exit(f"FAIL: anonymous WebSocket closed with {code}, expected 4401") + print("anonymous WebSocket rejected with close code 4401") + else: + sys.exit("FAIL: anonymous WebSocket stayed open") +except InvalidStatus as exc: + sys.exit(f"FAIL: anonymous WebSocket handshake returned HTTP {status_of(exc)}") +except Exception as exc: # noqa: BLE001 - report any handshake failure + sys.exit(f"FAIL: anonymous WebSocket check raised {exc!r}") + +# 2. With the browser session cookie the upgrade is accepted and bridged. +# DSH may not serve a WebSocket on '/', so an upstream close (1011) is +# reported but tolerated; a 4401 would mean the gateway lost the cookie. +try: + with connect_ws({"Cookie": cookie, "Origin": f"http://{gw}"}) as ws: + try: + ws.recv(timeout=10) + print("authenticated WebSocket accepted and streaming") + except TimeoutError: + print("authenticated WebSocket accepted (no immediate close)") + except ConnectionClosed as exc: + code = exc.rcvd.code if exc.rcvd else None + if code == 4401: + sys.exit( + "FAIL: authenticated WebSocket was treated as unauthenticated (4401)" + ) + print(f"authenticated WebSocket accepted; upstream closed with {code}") +except InvalidStatus as exc: + sys.exit(f"FAIL: authenticated WebSocket handshake returned HTTP {status_of(exc)}") +except Exception as exc: # noqa: BLE001 - report any handshake failure + sys.exit(f"FAIL: authenticated WebSocket check raised {exc!r}") +PY + pass "gateway WebSocket: anonymous rejected (4401), authenticated upgrade bridged" + else + echo "WARNING: python websockets not installed; skipping the WebSocket check" >&2 + fi echo "waiting for idle sweeper (~75s)..." for _ in $(seq 1 30); do - STATE=$(curl -s -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" \ + STATE=$(curl -s -b "$JAR" "http://$GW/users/bob" \ | sed -n 's/.*"state":"\([^"]*\)".*/\1/p') [ "$STATE" = "Suspended" ] && break sleep 5 @@ -113,16 +229,19 @@ if command -v docker >/dev/null && command -v kind >/dev/null; then pass "idle sweeper suspended bob" T0=$(date +%s) - curl -sf -m 150 -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \ + curl -sf -m 150 -b "$JAR" "http://$GW/u/bob/" >/dev/null \ || fail "wake-on-connect request failed" pass "wake-on-connect: request held and served after $(( $(date +%s) - T0 ))s" - curl -sf -X DELETE -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" >/dev/null \ + curl -sf -X DELETE -b "$JAR" -H "Origin: http://$GW" \ + "http://$GW/users/bob" >/dev/null \ || fail "gateway user deletion failed" kill "$PF" 2>/dev/null || true + rm -f "$JAR" + pass "gateway delete: bob removed" elif [ "${SKIP_GATEWAY:-}" = "1" ]; then echo "WARNING: gateway checks SKIPPED (SKIP_GATEWAY=1) — signup, auth," >&2 - echo "WARNING: proxy, idle-suspend and wake-on-connect were NOT validated" >&2 + echo "WARNING: proxy, WebSocket, idle-suspend and wake-on-connect were NOT validated" >&2 GATEWAY_SKIPPED=1 else fail "docker + kind are required for the gateway checks (set SKIP_GATEWAY=1 to run only the resource-layer checks)"