mirror of
https://github.com/sredevopsorg/deepseek-kubernetes-sandbox.git
synced 2026-11-03 01:13:24 +00:00
fbe886f3b9
* fix(gateway): grant patch on sandboxclaims for token rotation The SSO landing path calls K8sClient.patch_token(), which patches the `dsh-<user>` SandboxClaim's token-hash annotation whenever the browser has no valid gateway session: first visit, an expired or foreign cookie, or a gateway restart that drops the in-memory session store. The gateway Role only granted get/list/create/delete on sandboxclaims, so the patch returned 403 and, because ApiException is not handled, every such GET / answered a bare 500 "Internal Server Error". Grant the missing `patch` verb on extensions.agents.x-k8s.io/sandboxclaims, and update the least-privilege comment plus the two docs that enumerate the Role verbs. Verified with `make check` (versions, example layout, manifests) and `yamllint`; the 403 and the resulting 500 were reproduced from a running install and cleared by applying the equivalent Role patch (RBAC is evaluated per request, so no gateway restart is needed). * fix(gateway): return 503 instead of a bare 500 on Kubernetes API errors Every gateway route talks to the Kubernetes API, and ApiException used to be unhandled: Starlette's ServerErrorMiddleware turned it into a bare 500 "Internal Server Error" plus an ERROR traceback. That masked a missing RBAC verb as a client-visible crash on GET / (the SSO landing path), and it would equally hide an API outage or throttling. Register an ApiException handler that logs one credential-free line (status/reason) and answers 503 + Retry-After with a fixed body, so the Kubernetes Status payload (which can name cluster objects) is never proxied. Guard the first get_claim() in the WebSocket route the same way: WebSocket scopes bypass Starlette's HTTP exception handlers, so it now closes with 1011 instead of escaping as a traceback. Tests assert the 503 + Retry-After response, cover the SSO landing path that produced the reported 500 (forbidden claim patch), and assert the WebSocket close code. The failure-semantics tables document the new behaviour. --------- Co-authored-by: SREDevOpsAgents <agents@sredevops.org>
254 lines
11 KiB
YAML
254 lines
11 KiB
YAML
# The gateway: control+data plane that turns the resources into a usable
|
|
# browser-facing service. It is a Starlette ASGI app (see gateway/) listening
|
|
# on :8080 and exposing:
|
|
# GET /healthz unauthenticated health probe
|
|
# GET / landing page (signup / token login)
|
|
# POST /users, POST /login, ... session cookie auth
|
|
# /u/<user>/... HTTP + WebSocket proxy to the user's DSH
|
|
#
|
|
# Image: the release image published for this example, pinned by tag.
|
|
# imagePullPolicy: IfNotPresent lets a local kind build loaded under the same
|
|
# tag be used without a registry round-trip (scripts/e2e-kind.sh builds
|
|
# dsh-aaas-gateway:e2e and overrides this image).
|
|
#
|
|
# SSO IS OPT-IN: this manifest runs the gateway with its own bearer/session
|
|
# auth only, so AUTH_PROXY_HEADER is intentionally NOT set (see the env
|
|
# block). To put GitHub OAuth in front of it, use
|
|
# manifests/exposure/gateway-api-sso/.
|
|
#
|
|
# This Service is ClusterIP and is the only intended entry point: expose it
|
|
# through exactly ONE of the documented variants in manifests/exposure/
|
|
# (ingress/ or gateway-api-sso/), never both and never directly.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
---
|
|
# Least privilege:
|
|
# - claims are created/deleted per user (signup/delete) and patched only to
|
|
# rotate the token-hash annotation when a session is minted;
|
|
# - sandboxes are read, and patched only to flip operatingMode (suspend/resume);
|
|
# - pods are read, and pods/log is read to capture DSH's launch token from
|
|
# the startup log line (`dsh web: ...?token=...`).
|
|
# configmaps are not needed: all state is in memory / claim annotations.
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
rules:
|
|
- apiGroups: ["extensions.agents.x-k8s.io"]
|
|
resources: ["sandboxclaims"]
|
|
verbs: ["get", "list", "create", "patch", "delete"]
|
|
- apiGroups: ["agents.x-k8s.io"]
|
|
resources: ["sandboxes"]
|
|
verbs: ["get", "list", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: [""]
|
|
resources: ["pods/log"]
|
|
verbs: ["get"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: dsh-gateway
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
spec:
|
|
# Single replica BY DESIGN: the idle clock, in-flight counters and the
|
|
# per-(user, pod, Host) DSH cookie cache are in-process memory. Running two
|
|
# replicas would fragment that state (double suspend/resume races, repeated
|
|
# log-token handshakes). Scale only after moving state to a shared store.
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: dsh-gateway
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: dsh-gateway
|
|
spec:
|
|
serviceAccountName: dsh-gateway
|
|
# OPTIONAL: run the gateway under gVisor. Requires a RuntimeClass named
|
|
# "gvisor" and nodes that can run it. See docs/optional-hardening.md.
|
|
# runtimeClassName: gvisor
|
|
#
|
|
# OPTIONAL: pin the gateway to a dedicated node pool. Replace the
|
|
# placeholder with a label your nodes carry; add a toleration for a
|
|
# tainted pool. Keep active cluster-specific scheduling in a local patch.
|
|
# nodeSelector:
|
|
# dsh.example.com/node-pool: dsh
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10000 # matches USER in gateway/Dockerfile
|
|
runAsGroup: 10000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: gateway
|
|
image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox/dsh-agents-as-a-service/gateway:dsh-v0.2.1-rc.5-r6
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
env:
|
|
- name: NAMESPACE
|
|
value: dsh-demo
|
|
- name: POOL
|
|
value: dsh-pool
|
|
# Seconds a sandbox may sit idle before the sweeper suspends it
|
|
# (pod deleted, PVC kept), and the max time a proxy request waits
|
|
# for a suspended sandbox to become Ready again. Only claims this
|
|
# gateway created are swept: a claim applied by hand from the
|
|
# resource-layer manifests carries no aaas.example.com/managed-by
|
|
# annotation and is never suspended.
|
|
#
|
|
# These are the demo values (360/240). scripts/e2e-kind.sh
|
|
# overrides both to 60/120 with `kubectl set env` so the smoke
|
|
# test does not wait six minutes for the sweeper.
|
|
- name: IDLE_TIMEOUT
|
|
value: "360"
|
|
- name: WAKE_TIMEOUT
|
|
value: "240"
|
|
# Container name inside the per-user sandbox pods whose logs carry
|
|
# the DSH launch token.
|
|
- name: DSH_CONTAINER
|
|
value: dsh-web
|
|
# Browser session cookie: HttpOnly + SameSite=Lax, holds only an
|
|
# opaque, server-side session id (sid) - never the user's DSH
|
|
# token. Sessions can be revoked server-side: POST /logout, user
|
|
# delete, POST /recreate and expiry (COOKIE_MAX_AGE, default
|
|
# 7 days) all invalidate them. The session table is in process
|
|
# memory, so restarting this Deployment logs every browser out.
|
|
# With COOKIE_SECURE=true the cookie is named
|
|
# __Host-dsh_gw_session; COOKIE_NAME overrides the name.
|
|
#
|
|
# "false" keeps the plain-HTTP demo usable through port-forward: a
|
|
# Secure cookie is never sent over HTTP, so login would silently
|
|
# fail. The gateway REFUSES TO START with COOKIE_SECURE=false
|
|
# unless ALLOW_INSECURE_COOKIES=1 acknowledges that the session
|
|
# travels in clear text, so the base demo sets both. HTTPS
|
|
# deployments (the ingress/ variant with TLS, or gateway-api-sso/,
|
|
# which terminates TLS at the Gateway) MUST set this to "true".
|
|
- name: COOKIE_SECURE
|
|
value: "false"
|
|
# Explicit acknowledgement that this base demo serves plain HTTP.
|
|
# Required when COOKIE_SECURE=false (the gateway exits at startup
|
|
# otherwise); ignored once COOKIE_SECURE=true, so HTTPS installs
|
|
# should drop it. Never set this on an internet-facing deployment.
|
|
- name: ALLOW_INSECURE_COOKIES
|
|
value: "1"
|
|
# SSO IS OPT-IN AND DISABLED HERE. With AUTH_PROXY_HEADER unset the
|
|
# gateway trusts only its own bearer/session auth and ignores every
|
|
# identity header. Enable it ONLY behind
|
|
# manifests/exposure/gateway-api-sso/, where oauth2-proxy is the
|
|
# sole route to this Service: set it to X-Forwarded-User, keep the
|
|
# Service ClusterIP and reachable only through the proxy, and
|
|
# terminate TLS in front of the gateway. Otherwise anyone who can
|
|
# reach :8080 can impersonate any user by sending that header.
|
|
# - name: AUTH_PROXY_HEADER
|
|
# value: X-Forwarded-User
|
|
# Brute-force / abuse throttle for the unauthenticated auth
|
|
# endpoints (/users and /login): at most N attempts per client per
|
|
# AUTH_RATE_WINDOW seconds (default 60). A client is the rightmost
|
|
# X-Forwarded-For entry (the one added by the trusted proxy) when
|
|
# present, otherwise the TCP peer. Over the limit the gateway
|
|
# answers 429 + Retry-After BEFORE reading the body or touching the
|
|
# cluster; 0 disables the limiter. This is a per-process bucket, so
|
|
# it is not a substitute for an ingress rate limit.
|
|
- name: AUTH_RATE_LIMIT
|
|
value: "30"
|
|
# Window in seconds for AUTH_RATE_LIMIT (default 60).
|
|
# - name: AUTH_RATE_WINDOW
|
|
# value: "60"
|
|
# Optional signup gate. When set, POST /users requires the invite
|
|
# code - form field "invite" or header X-Signup-Invite - compared in
|
|
# constant time. A missing/wrong code is rejected with 403 before
|
|
# the user is looked up, so it also stops unauthenticated claim
|
|
# creation. The landing page shows the invite input only when this
|
|
# is set. Keep the value in a Secret, create it before rollout and
|
|
# restart the gateway to pick up a rotation:
|
|
# kubectl -n dsh-demo create secret generic dsh-signup-invite \
|
|
# --from-literal=code="$(openssl rand -hex 16)"
|
|
# - name: SIGNUP_INVITE_CODE
|
|
# valueFrom:
|
|
# secretKeyRef:
|
|
# name: dsh-signup-invite
|
|
# key: code
|
|
# Reject proxied request bodies larger than this (413).
|
|
- name: MAX_BODY_MB
|
|
value: "32"
|
|
# Per-user concurrency guard (default 32): at most this many
|
|
# proxied HTTP requests and WebSocket connections per user.
|
|
# Excess HTTP requests get 503 + Retry-After and excess WebSocket
|
|
# upgrades close with 1011, so one tenant cannot exhaust the
|
|
# single-replica gateway for everyone else.
|
|
# - name: MAX_IN_FLIGHT_PER_USER
|
|
# value: "32"
|
|
# Largest WebSocket message received from DSH, in bytes (default
|
|
# 16 MiB); oversized frames close the connection. Browser-to-DSH
|
|
# frames are bounded by uvicorn's own limit. HTTP/SSE responses are
|
|
# streamed without this cap.
|
|
# - name: WS_MAX_MESSAGE_BYTES
|
|
# value: "16777216"
|
|
# /healthz is unauthenticated and returns 200 without touching the
|
|
# cluster, so it is safe for both readiness and liveness.
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: 8080
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
failureThreshold: 3
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: 8080
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
failureThreshold: 3
|
|
resources:
|
|
requests: {cpu: 50m, memory: 128Mi}
|
|
limits: {cpu: 500m, memory: 256Mi}
|
|
volumeMounts:
|
|
# readOnlyRootFilesystem leaves no writable HOME/tmp; the image sets
|
|
# HOME=/tmp, so back it with scratch space. No state lives here.
|
|
- name: tmp
|
|
mountPath: /tmp
|
|
volumes:
|
|
- name: tmp
|
|
emptyDir: {}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: dsh-gateway
|
|
namespace: dsh-demo
|
|
spec:
|
|
selector:
|
|
app: dsh-gateway
|
|
ports:
|
|
- port: 8080
|
|
targetPort: 8080
|