mirror of
https://github.com/sredevopsorg/deepseek-kubernetes-sandbox.git
synced 2026-11-03 01:13:24 +00:00
Make DSH agents-as-a-service browser-usable; fix probe crash-loop (#3)
* Rewrite DSH gateway for browser sessions, HTTP and WebSocket - Replace the Flask proxy with a Starlette ASGI app (uvicorn factory), keeping the control plane (claims, wake/suspend, idle sweeper). - Add HttpOnly gateway sessions and a server-side DSH launch-token handshake: read the token from pod logs, exchange it for a dsh-auth cookie, cache per (user, pod UID, Host), refresh once on 401. - Proxy HTTP/SSE and WebSocket (/api/remote.mux) under /u/<user>/, with header stripping/injection, Location rewriting and wake-on-connect. - Add a landing/signup/login page, health endpoint, pinned dependencies, a read-only-rootfs gateway image, 87 cluster-free tests and docs. * Fix DSH probe crash-loop and harden gateway manifests - Probe /favicon.svg instead of /: DSH gates / behind its launch-token cookie and returns 401, which made warm-pool pods crash-loop. - Pass DSH_TRUSTED_HOSTS through the entrypoint as --trusted-host args for public/ingress hosts; keep localhost trusted. - Deploy the published :gateway image, add pods/pods-log RBAC, /healthz probes, read-only rootfs with a /tmp emptyDir and non-root security. - Add optional Traefik Ingress and document the prerequisites. * Update CI, smoke tests, build tooling and docs - Run gateway pytest/ruff and build/smoke-test the image in CI. - Parameterize the DSH build version and pnpm version inputs. - Exercise cookie signup, HTML proxy, WebSocket auth, suspend/wake and delete in run-test-kind.sh. - Add Makefile/build.sh targets and document the architecture, handshake, probes, trusted hosts, troubleshooting and limitations. --------- Co-authored-by: SREDevOpsAgents <agents@sredevops.org>
This commit is contained in:
SREDevOps.org Agent
committed by
GitHub
GitHub
parent
c9be929212
commit
6301c21bea
@@ -12,7 +12,11 @@ cd "$(dirname "$0")"
|
||||
NS=dsh-demo
|
||||
pass() { echo "PASS: $1"; }
|
||||
fail() { echo "FAIL: $1" >&2; exit 1; }
|
||||
cleanup() { kubectl delete namespace "$NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true; }
|
||||
cleanup() {
|
||||
kill "${PF:-}" 2>/dev/null || true
|
||||
rm -f "${JAR:-}"
|
||||
kubectl delete namespace "$NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "=== 1. install prereqs (throwaway key) + template + warm pool"
|
||||
@@ -77,34 +81,146 @@ BOUND=$(kubectl -n "$NS" get sandboxclaim dsh-mallory -o jsonpath='{.status.sand
|
||||
kubectl delete -f 40-claim-rejected.yaml >/dev/null
|
||||
pass "claim rejected with EnvVarsInjectionRejected and never bound"
|
||||
|
||||
echo "=== 7. gateway: signup, proxy, idle-suspend, wake-on-connect"
|
||||
echo "=== 7. gateway: browser signup, proxy, idle-suspend, wake-on-connect"
|
||||
if command -v docker >/dev/null && command -v kind >/dev/null; then
|
||||
KIND_CLUSTER="${KIND_CLUSTER:-$(kubectl config current-context | sed 's/^kind-//')}"
|
||||
docker build -q -t dsh-aaas-gateway:demo gateway/ >/dev/null
|
||||
kind load docker-image dsh-aaas-gateway:demo --name "$KIND_CLUSTER" >/dev/null
|
||||
# Tag exactly what 50-gateway.yaml references so the smoke test exercises
|
||||
# the shipped manifest unchanged (imagePullPolicy: IfNotPresent uses the
|
||||
# image loaded into kind).
|
||||
GW_IMAGE=ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:gateway
|
||||
docker build -q -t "$GW_IMAGE" gateway/ >/dev/null
|
||||
kind load docker-image "$GW_IMAGE" --name "$KIND_CLUSTER" >/dev/null
|
||||
kubectl apply -f 50-gateway.yaml
|
||||
kubectl -n "$NS" rollout status deploy/dsh-gateway --timeout=120s
|
||||
# Local port-forward uses 18080 (NOT 3080, which stays in-cluster).
|
||||
kubectl -n "$NS" port-forward svc/dsh-gateway 18080:8080 >/dev/null 2>&1 &
|
||||
PF=$!
|
||||
sleep 3
|
||||
GW=localhost:18080
|
||||
JAR=$(mktemp)
|
||||
|
||||
SIGNUP=$(curl -s -X POST "http://$GW/users" -H 'Content-Type: application/json' -d '{"user":"bob"}')
|
||||
TOKEN=$(echo "$SIGNUP" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')
|
||||
[ -n "$TOKEN" ] || fail "signup returned no token: $SIGNUP"
|
||||
echo "$SIGNUP" | grep -q '"state":"Ready"' || fail "signup not Ready: $SIGNUP"
|
||||
pass "gateway signup: bob Ready with one-time token"
|
||||
for _ in $(seq 1 20); do
|
||||
HEALTH=$(curl -s -o /dev/null -w '%{http_code}' "http://$GW/healthz" || true)
|
||||
[ "$HEALTH" = "200" ] && break
|
||||
sleep 1
|
||||
done
|
||||
[ "${HEALTH:-}" = "200" ] || fail "gateway /healthz was not reachable on $GW"
|
||||
pass "gateway health: /healthz -> 200"
|
||||
|
||||
curl -sf -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \
|
||||
|| fail "proxied web call failed"
|
||||
curl -s -o /dev/null -w '%{http_code}' "http://$GW/u/bob/" | grep -q 401 \
|
||||
|| fail "tokenless request was not rejected"
|
||||
pass "gateway proxy: authed request 200, tokenless 401"
|
||||
# Signup exactly as the landing page does it: a form POST, answered with a
|
||||
# session cookie. The browser never handles the DSH launch token.
|
||||
SIGNUP_CODE=$(curl -s -o /tmp/dsh-gw-signup.out -w '%{http_code}' \
|
||||
-c "$JAR" -H "Origin: http://$GW" \
|
||||
-X POST "http://$GW/users" -d 'user=bob')
|
||||
case "$SIGNUP_CODE" in
|
||||
201|202|303) ;;
|
||||
*) fail "signup returned HTTP $SIGNUP_CODE: $(cat /tmp/dsh-gw-signup.out)" ;;
|
||||
esac
|
||||
grep -q 'dsh_gw_session' "$JAR" || fail "signup did not set the dsh_gw_session cookie"
|
||||
STATE=""
|
||||
for _ in $(seq 1 30); do
|
||||
STATE=$(curl -s -b "$JAR" "http://$GW/users/bob" \
|
||||
| sed -n 's/.*"state":"\([^"]*\)".*/\1/p')
|
||||
[ "$STATE" = "Ready" ] && break
|
||||
sleep 2
|
||||
done
|
||||
[ "$STATE" = "Ready" ] || fail "bob's claim was not Ready (state=$STATE)"
|
||||
pass "gateway signup: bob Ready with an HttpOnly session cookie"
|
||||
|
||||
CT=$(curl -s -o /tmp/dsh-u-bob.html -w '%{content_type}' -b "$JAR" "http://$GW/u/bob/")
|
||||
case "$CT" in text/html*) ;; *) fail "/u/bob/ returned content-type '$CT', expected text/html";; esac
|
||||
grep -qi '<html' /tmp/dsh-u-bob.html || fail "/u/bob/ did not return the DSH Web HTML document"
|
||||
pass "gateway proxy: /u/bob/ returns DSH Web HTML through path routing"
|
||||
|
||||
CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://$GW/u/bob/")
|
||||
[ "$CODE" = "401" ] || fail "tokenless /u/bob/ returned $CODE, expected 401"
|
||||
pass "gateway proxy: tokenless request rejected with 401"
|
||||
|
||||
# Optional WebSocket check: the gateway must reject upgrades without its
|
||||
# session cookie and bridge authenticated upgrades to the sandbox.
|
||||
if python3 -c 'import websockets' >/dev/null 2>&1; then
|
||||
python3 - "$JAR" "$GW" <<'PY' || fail "WebSocket checks failed"
|
||||
import sys
|
||||
|
||||
from websockets.exceptions import ConnectionClosed, InvalidStatus
|
||||
from websockets.sync.client import connect
|
||||
|
||||
jar_path, gw = sys.argv[1], sys.argv[2]
|
||||
cookie = None
|
||||
with open(jar_path, encoding="utf-8") as jar:
|
||||
for raw in jar:
|
||||
line = raw.removeprefix("#HttpOnly_")
|
||||
if not line.strip() or line.startswith("#"):
|
||||
continue
|
||||
fields = line.rstrip("\n").split("\t")
|
||||
if len(fields) >= 7 and fields[5] == "dsh_gw_session":
|
||||
cookie = f"{fields[5]}={fields[6]}"
|
||||
break
|
||||
if not cookie:
|
||||
sys.exit("FAIL: no dsh_gw_session cookie in the jar")
|
||||
|
||||
uri = f"ws://{gw}/u/bob/api/remote.mux"
|
||||
|
||||
|
||||
def connect_ws(headers):
|
||||
try:
|
||||
return connect(uri, additional_headers=headers, open_timeout=30)
|
||||
except TypeError: # websockets < 14
|
||||
return connect(uri, extra_headers=headers, open_timeout=30)
|
||||
|
||||
|
||||
def status_of(exc):
|
||||
response = getattr(exc, "response", None)
|
||||
return getattr(response, "status_code", None) or getattr(exc, "status_code", None)
|
||||
|
||||
|
||||
# 1. Without the gateway cookie the gateway accepts the upgrade and then
|
||||
# closes with application code 4401 (WebSocket auth has no HTTP 401 path).
|
||||
try:
|
||||
with connect(uri, open_timeout=10) as ws:
|
||||
try:
|
||||
ws.recv(timeout=10)
|
||||
except ConnectionClosed as exc:
|
||||
code = exc.rcvd.code if exc.rcvd else None
|
||||
if code != 4401:
|
||||
sys.exit(f"FAIL: anonymous WebSocket closed with {code}, expected 4401")
|
||||
print("anonymous WebSocket rejected with close code 4401")
|
||||
else:
|
||||
sys.exit("FAIL: anonymous WebSocket stayed open")
|
||||
except InvalidStatus as exc:
|
||||
sys.exit(f"FAIL: anonymous WebSocket handshake returned HTTP {status_of(exc)}")
|
||||
except Exception as exc: # noqa: BLE001 - report any handshake failure
|
||||
sys.exit(f"FAIL: anonymous WebSocket check raised {exc!r}")
|
||||
|
||||
# 2. With the browser session cookie the upgrade is accepted and bridged.
|
||||
# DSH may not serve a WebSocket on '/', so an upstream close (1011) is
|
||||
# reported but tolerated; a 4401 would mean the gateway lost the cookie.
|
||||
try:
|
||||
with connect_ws({"Cookie": cookie, "Origin": f"http://{gw}"}) as ws:
|
||||
try:
|
||||
ws.recv(timeout=10)
|
||||
print("authenticated WebSocket accepted and streaming")
|
||||
except TimeoutError:
|
||||
print("authenticated WebSocket accepted (no immediate close)")
|
||||
except ConnectionClosed as exc:
|
||||
code = exc.rcvd.code if exc.rcvd else None
|
||||
if code == 4401:
|
||||
sys.exit(
|
||||
"FAIL: authenticated WebSocket was treated as unauthenticated (4401)"
|
||||
)
|
||||
print(f"authenticated WebSocket accepted; upstream closed with {code}")
|
||||
except InvalidStatus as exc:
|
||||
sys.exit(f"FAIL: authenticated WebSocket handshake returned HTTP {status_of(exc)}")
|
||||
except Exception as exc: # noqa: BLE001 - report any handshake failure
|
||||
sys.exit(f"FAIL: authenticated WebSocket check raised {exc!r}")
|
||||
PY
|
||||
pass "gateway WebSocket: anonymous rejected (4401), authenticated upgrade bridged"
|
||||
else
|
||||
echo "WARNING: python websockets not installed; skipping the WebSocket check" >&2
|
||||
fi
|
||||
|
||||
echo "waiting for idle sweeper (~75s)..."
|
||||
for _ in $(seq 1 30); do
|
||||
STATE=$(curl -s -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" \
|
||||
STATE=$(curl -s -b "$JAR" "http://$GW/users/bob" \
|
||||
| sed -n 's/.*"state":"\([^"]*\)".*/\1/p')
|
||||
[ "$STATE" = "Suspended" ] && break
|
||||
sleep 5
|
||||
@@ -113,16 +229,19 @@ if command -v docker >/dev/null && command -v kind >/dev/null; then
|
||||
pass "idle sweeper suspended bob"
|
||||
|
||||
T0=$(date +%s)
|
||||
curl -sf -m 150 -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \
|
||||
curl -sf -m 150 -b "$JAR" "http://$GW/u/bob/" >/dev/null \
|
||||
|| fail "wake-on-connect request failed"
|
||||
pass "wake-on-connect: request held and served after $(( $(date +%s) - T0 ))s"
|
||||
|
||||
curl -sf -X DELETE -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" >/dev/null \
|
||||
curl -sf -X DELETE -b "$JAR" -H "Origin: http://$GW" \
|
||||
"http://$GW/users/bob" >/dev/null \
|
||||
|| fail "gateway user deletion failed"
|
||||
kill "$PF" 2>/dev/null || true
|
||||
rm -f "$JAR"
|
||||
pass "gateway delete: bob removed"
|
||||
elif [ "${SKIP_GATEWAY:-}" = "1" ]; then
|
||||
echo "WARNING: gateway checks SKIPPED (SKIP_GATEWAY=1) — signup, auth," >&2
|
||||
echo "WARNING: proxy, idle-suspend and wake-on-connect were NOT validated" >&2
|
||||
echo "WARNING: proxy, WebSocket, idle-suspend and wake-on-connect were NOT validated" >&2
|
||||
GATEWAY_SKIPPED=1
|
||||
else
|
||||
fail "docker + kind are required for the gateway checks (set SKIP_GATEWAY=1 to run only the resource-layer checks)"
|
||||
|
||||
Reference in New Issue
Block a user