nicolas-georger/deepseek-kubernetes-sandbox
1
0
Fork 0
mirror of https://github.com/sredevopsorg/deepseek-kubernetes-sandbox.git synced 2026-11-03 01:13:24 +00:00
Code Projects Releases Activity

fix(gateway)!: replace token cookies with revocable sessions and harden abuse controls (#40)

Browse Source 
- Replace user:token cookies with opaque server-side sessions, revocable on logout, delete, recreate and expiry; bind every credential to the path user and add cross-user regression tests.
- Rate-limit /users and /login, add optional signup invite codes and form same-origin checks, and unify login failures to 401.
- Require explicit ALLOW_INSECURE_COOKIES for plain HTTP, use __Host-dsh_gw_session when secure, and fix secure cookie deletion.
- Strip client-supplied forwarding/SSO headers, add an optional upstream allowlist, per-user in-flight and WebSocket message caps, credential-free handshake errors, and sweep-time state pruning.
- Digest-pin base images, hash-pin gateway dependencies, verify DSH_COMMIT in the build, and run the gateway image as non-root with a pinned Python base.
- Update manifests, production checklist, architecture/security docs and add an SSO preflight script.

BREAKING CHANGE: existing dsh_gw_session=<user>:<token> cookies are invalid; users must sign in again. Unknown users on POST /login now return 401 instead of 404, and the gateway refuses COOKIE_SECURE=false unless ALLOW_INSECURE_COOKIES=1.

Co-authored-by: SREDevOpsAgents <agents@sredevops.org>
This commit is contained in:
SREDevOps.org Agent
2026-10-31 00:57:35 -03:00
committed by GitHub
parent 2f959e6afb
commit 1a929e3e24
33 changed files with 5450 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.env.sample
View File

@@ -23,6 +23,9 @@ CEREBRAS_API_KEY=
AI_GATEWAY_API_KEY=
HF_TOKEN=
DSH_VERSION=dsh-v0.2.1-rc.5
# Full upstream commit SHA that DSH_VERSION resolves to; the main image build
# fails if the clone's HEAD differs (guards against a moved tag).
DSH_COMMIT=b830ec950faba0a46dda564be3e2d854afed94f1
RELEASE_REVISION=
PNPM_VERSION=11.28.5
# Container port, and the host port published by "make run".

3
.github/workflows/build-main.yaml vendored
View File

@@ -29,6 +29,7 @@ jobs:
outputs:
identity: ${{ steps.meta.outputs.identity }}
dsh_version: ${{ steps.meta.outputs.dsh_version }}
dsh_commit: ${{ steps.meta.outputs.dsh_commit }}
pnpm_version: ${{ steps.meta.outputs.pnpm_version }}
release_revision: ${{ steps.meta.outputs.release_revision }}
ghcr_image: ${{ steps.meta.outputs.ghcr_image }}
@@ -53,6 +54,7 @@ jobs:
{
echo "identity=$identity"
echo "dsh_version=$DSH_VERSION"
echo "dsh_commit=$DSH_COMMIT"
echo "pnpm_version=$PNPM_VERSION"
echo "release_revision=$RELEASE_REVISION"
echo "ghcr_image=$GHCR_IMAGE"
@@ -75,6 +77,7 @@ jobs:
dockerfile: Dockerfile
build-args: |
DSH_VERSION=${{ needs.meta.outputs.dsh_version }}
DSH_COMMIT=${{ needs.meta.outputs.dsh_commit }}
RELEASE_REVISION=${{ needs.meta.outputs.release_revision }}
PNPM_VERSION=${{ needs.meta.outputs.pnpm_version }}
VCS_REF=${{ github.sha }}

2
.github/workflows/ci-image.yaml vendored
View File

@@ -48,6 +48,7 @@ jobs:
{
echo "identity=$identity"
echo "dsh_version=$DSH_VERSION"
echo "dsh_commit=$DSH_COMMIT"
echo "pnpm_version=$PNPM_VERSION"
echo "release_revision=${RELEASE_REVISION:-}"
} >> "$GITHUB_OUTPUT"
@@ -66,6 +67,7 @@ jobs:
tags: dsh-web:ci
build-args: |
DSH_VERSION=${{ steps.meta.outputs.dsh_version }}
DSH_COMMIT=${{ steps.meta.outputs.dsh_commit }}
RELEASE_REVISION=${{ steps.meta.outputs.release_revision }}
PNPM_VERSION=${{ steps.meta.outputs.pnpm_version }}
VCS_REF=${{ github.sha }}

4
.github/workflows/ci-python.yaml vendored
View File

@@ -43,7 +43,9 @@ jobs:
run: |
set -euo pipefail
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-dev.txt
# Both files are fully hash-pinned (uv pip compile --generate-hashes);
# pip refuses to install anything not listed with a matching hash.
pip install --require-hashes -r requirements.txt -r requirements-dev.txt
- name: Lint
run: |
set -euo pipefail

21
.github/workflows/release.yaml vendored
View File

@@ -34,6 +34,7 @@ jobs:
outputs:
identity: ${{ steps.meta.outputs.identity }}
dsh_version: ${{ steps.meta.outputs.dsh_version }}
dsh_commit: ${{ steps.meta.outputs.dsh_commit }}
pnpm_version: ${{ steps.meta.outputs.pnpm_version }}
release_revision: ${{ steps.meta.outputs.release_revision }}
ghcr_image: ${{ steps.meta.outputs.ghcr_image }}
@@ -91,13 +92,28 @@ jobs:
echo "tag '$tag' does not exist in this repository" >&2
exit 1
fi
if ! git ls-remote --tags --exit-code "$DSH_REPO" "$DSH_VERSION" >/dev/null; then
if ! upstream="$(git ls-remote --tags --exit-code "$DSH_REPO" \
"refs/tags/$DSH_VERSION^{}" "refs/tags/$DSH_VERSION")"; then
echo "upstream ref '$DSH_VERSION' not found in $DSH_REPO" >&2
exit 1
fi
# Never release a ref that no longer resolves to the pinned commit: an
# annotated tag has a peeled ^{} line, a lightweight tag only the ref.
resolved="$(printf '%s\n' "$upstream" \
| awk -v t="refs/tags/$DSH_VERSION^{}" '$2 == t {print $1}' | head -n1)"
if [ -z "$resolved" ]; then
resolved="$(printf '%s\n' "$upstream" \
| awk -v t="refs/tags/$DSH_VERSION" '$2 == t {print $1}' | head -n1)"
fi
if [ "$resolved" != "$DSH_COMMIT" ]; then
echo "upstream $DSH_REPO@$DSH_VERSION resolved to '$resolved'," \
"versions.env pins DSH_COMMIT=$DSH_COMMIT" >&2
exit 1
fi
{
echo "identity=$identity"
echo "dsh_version=$DSH_VERSION"
echo "dsh_commit=$DSH_COMMIT"
echo "pnpm_version=$PNPM_VERSION"
echo "release_revision=$RELEASE_REVISION"
echo "ghcr_image=$GHCR_IMAGE"
@@ -139,7 +155,7 @@ jobs:
run: |
set -euo pipefail
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-dev.txt
pip install --require-hashes -r requirements.txt -r requirements-dev.txt
- name: Lint
if: needs.meta.outputs.run_tests == 'true'
run: |
@@ -166,6 +182,7 @@ jobs:
dockerfile: Dockerfile
build-args: |
DSH_VERSION=${{ needs.meta.outputs.dsh_version }}
DSH_COMMIT=${{ needs.meta.outputs.dsh_commit }}
RELEASE_REVISION=${{ needs.meta.outputs.release_revision }}
PNPM_VERSION=${{ needs.meta.outputs.pnpm_version }}
VCS_REF=${{ github.sha }}

28
AGENTS.md
View File

@@ -117,17 +117,33 @@ needs kind plus the agent-sandbox CRDs (see the example README).
- AUTH_PROXY_HEADER is opt-in. Only enable it when the gateway Service is
reachable exclusively through the authenticating proxy; otherwise any client
can spoof X-Forwarded-User.
can spoof X-Forwarded-User. The SSO variant's NetworkPolicy is required for
production, and `examples/dsh-agents-as-a-service/scripts/preflight-sso.sh`
verifies the setup (Secrets, gateway env, TLS reference, placeholders, second
paths).
- The example gateway reads the DSH launch token from pod logs. Treat pod logs
as sensitive and keep its RBAC namespace-scoped.
- Browser sessions are opaque server-side ids held in gateway memory: they are
revocable (logout/delete/recreate/expiry) but a gateway restart logs everyone
out and horizontal scaling needs a shared store. Session cookies are never
accepted as bearer tokens.
- AUTH_RATE_LIMIT, SIGNUP_INVITE_CODE, MAX_IN_FLIGHT_PER_USER and
WS_MAX_MESSAGE_BYTES are in-process abuse guards, not ingress rate limits,
quotas or an audit log. Add those for an internet-facing deployment.
- The example shares provider keys (dsh-provider-keys plus the optional
dsh-provider-keys-extra) across sandboxes; do not describe it as
tenant-isolated.
- gVisor, nodeSelector, the NetworkPolicies, extra provider keys and failed-Pod
recovery are opt-in and are not active in the base manifests; do not describe
the base example as hardened.
- TLS terminates outside the gateway; set COOKIE_SECURE=true for HTTPS and keep
the base manifest safe for local HTTP.
- gVisor, nodeSelector, the shipped sandbox/exposure NetworkPolicies and the
documented gateway egress policy, extra provider keys and failed-Pod recovery
are opt-in and are not active in the base manifests; do not describe the base
example as hardened.
The sandbox and gateway-egress policies are required production steps; an
exposure without its NetworkPolicy is demo-only. See the example's
production checklist.
- TLS terminates outside the gateway. Set COOKIE_SECURE=true for HTTPS and drop
ALLOW_INSECURE_COOKIES; the gateway refuses COOKIE_SECURE=false unless
ALLOW_INSECURE_COOKIES=1, which is only for the plain-HTTP local demo. Never
expose that combination.
## Ask before

20
Dockerfile
View File

@@ -6,13 +6,14 @@
# Build args (all pinned in versions.env, verified by scripts/check-versions.sh):
# DSH_REPO upstream git remote to package
# DSH_VERSION upstream git ref to build (tag or branch)
# DSH_COMMIT full commit SHA DSH_VERSION must resolve to
# RELEASE_REVISION optional repository revision label (r2, r3, ...)
# PNPM_VERSION pnpm used to build DSH
# VCS_REF this repository's commit, recorded in OCI labels
# NODE_BUILD_IMAGE node image used to build DSH
# NODE_RUNTIME_IMAGE node image used at runtime
ARG NODE_BUILD_IMAGE=docker.io/node:24-trixie
ARG NODE_RUNTIME_IMAGE=docker.io/node:24-trixie-slim
# NODE_BUILD_IMAGE node image used to build DSH (pinned by digest)
# NODE_RUNTIME_IMAGE node image used at runtime (pinned by digest)
ARG NODE_BUILD_IMAGE=docker.io/node:24-trixie@sha256:8c761eca2bf7eec965f946cf47b8090143cb41f45f2f8699af0cc0298d2d2d19
ARG NODE_RUNTIME_IMAGE=docker.io/node:24-trixie-slim@sha256:54f84b4cf8432b21aaffd5d3ce8d41e12cea0d10d33c09d2f3c9879d49d6b446
# ---------------------------------------------------------------------------
# Build stage: clone DSH at the pinned ref, build, and package the CLI.
@@ -21,6 +22,7 @@ FROM $NODE_BUILD_IMAGE AS build
ARG DSH_REPO=https://github.com/deepseek-ai/deepseek-harness.git
ARG DSH_VERSION=dsh-v0.2.1-rc.5
ARG DSH_COMMIT=b830ec950faba0a46dda564be3e2d854afed94f1
ARG PNPM_VERSION=11.28.5
ENV CI=true \
@@ -31,7 +33,15 @@ RUN corepack enable \
WORKDIR /src
RUN git clone --depth 1 --branch "$DSH_VERSION" "$DSH_REPO" .
# Clone the pinned ref and fail closed unless it resolves to DSH_COMMIT. A moved
# tag would otherwise silently change what the image packages.
RUN git clone --depth 1 --branch "$DSH_VERSION" "$DSH_REPO" . \
&& actual="$(git -C /src rev-parse HEAD)" \
&& if [ "$actual" != "$DSH_COMMIT" ]; then \
echo "error: $DSH_REPO $DSH_VERSION resolved to $actual, expected DSH_COMMIT $DSH_COMMIT" >&2; \
echo "error: update DSH_VERSION/DSH_COMMIT in versions.env with scripts/set-version.sh" >&2; \
exit 1; \
fi
RUN pnpm install --frozen-lockfile

1
Makefile
View File

@@ -27,6 +27,7 @@ help: ## Show available targets
image: ## Build the main DSH Web image (no push)
docker build \
--build-arg DSH_VERSION=$(DSH_VERSION) \
--build-arg DSH_COMMIT=$(DSH_COMMIT) \
--build-arg RELEASE_REVISION=$(RELEASE_REVISION) \
--build-arg PNPM_VERSION=$(PNPM_VERSION) \
--build-arg VCS_REF=$(VCS_REF) \

38
SECURITY.md
View File

@@ -22,22 +22,40 @@ disclosure once a fix is available.
tenant. Keep the gateway's RBAC namespace-scoped and treat pod logs as
sensitive. Replacing log scraping with an upstream token API is tracked in
the [example architecture document](examples/dsh-agents-as-a-service/docs/architecture.md#upstream-feature-request-remove-log-scraping).
- **Browser sessions are server-side.** The gateway session cookie holds an
opaque session id, not a user name or token; it is not accepted as a bearer
token. Sessions are revoked on logout, user delete, `/recreate` and expiry,
but they live in gateway memory, so a gateway restart logs every browser out
and scaling beyond one replica requires a shared store. Do not treat the
cookie as durable or cross-replica state.
- **SSO header trust boundary.** AUTH_PROXY_HEADER makes the gateway trust an
identity header (for example X-Forwarded-User). Enable it only when the
gateway Service is reachable exclusively through the authenticating proxy;
otherwise any client can spoof the header. The SSO exposure variant ships a
NetworkPolicy that restricts gateway access to oauth2-proxy pods; a custom
exposure needs an equivalent.
gateway Service is reachable exclusively through the authenticating proxy
and its NetworkPolicy; otherwise any client can spoof the header. The SSO
exposure variant's policy is required, not optional: an exposure without it
is demo-only. Run `examples/dsh-agents-as-a-service/scripts/preflight-sso.sh`
after applying the variant; it checks the Secrets, gateway env, TLS
reference, leftover placeholders and accidental second paths.
- **Insecure-cookie guard.** TLS terminates outside the gateway. Set
COOKIE_SECURE=true for HTTPS and remove ALLOW_INSECURE_COOKIES; the gateway
refuses to start with COOKIE_SECURE=false unless ALLOW_INSECURE_COOKIES=1 is
explicitly set, which exists only for the plain-HTTP local demo. Never expose
that combination outside localhost.
- **Abuse controls are in-process.** AUTH_RATE_LIMIT/AUTH_RATE_WINDOW,
SIGNUP_INVITE_CODE, MAX_IN_FLIGHT_PER_USER and WS_MAX_MESSAGE_BYTES bound
casual abuse and resource exhaustion on the single gateway replica. They
reset on restart and are not an ingress rate limiter, WAF, quota system or
audit log; add those for an internet-facing deployment.
- **Shared provider keys.** The example injects DEEPSEEK_API_KEY and any keys in
the optional dsh-provider-keys-extra Secret into every tenant sandbox (a
warm-pool constraint). Treat tenants as mutually trusting until per-tenant
credentials or an egress proxy exist.
- **Optional isolation is opt-in.** gVisor, nodeSelector, the sandbox/exposure
NetworkPolicies and extra provider keys are opt-in. Without a
NetworkPolicy-capable CNI or a gVisor runtime, the base example is not
tenant-isolated; see examples/dsh-agents-as-a-service/docs/optional-hardening.md.
- **TLS is external.** Terminate TLS at an Ingress or Gateway and set
COOKIE_SECURE=true. Plain HTTP outside localhost exposes session cookies.
- **The base example is demo-only, not hardened.** gVisor, nodeSelector, the
sandbox/exposure NetworkPolicies, the gateway egress policy and extra provider
keys are opt-in. Without gVisor and the network policies there is no
tenant-to-tenant isolation; the sandboxes share the host kernel and can reach
cluster-internal endpoints. Production requirements are enumerated in the
[production checklist](examples/dsh-agents-as-a-service/docs/optional-hardening.md#6-production-checklist).
- **No secrets in the repository.** The repository only ships empty Secret
manifests and *.yaml.example files.

21
docs/main-image.md
View File

@@ -11,12 +11,16 @@ The image is image-first: it is the product, and the examples under
## What the build does
The Dockerfile is a two-stage build:
The Dockerfile is a two-stage build. Both stages use the `node` images pinned by
digest in [versions.env](../versions.env): a mutable tag alone is not trusted, and
`./scripts/check-versions.sh` rejects an image reference without a `@sha256:` digest.
1. **Build stage** (`node:24-trixie`) clones DSH at the pinned git ref, installs
1. **Build stage** (`NODE_BUILD_IMAGE`) clones DSH at the pinned git ref, verifies
that the clone's `HEAD` is exactly `DSH_COMMIT` (the build fails otherwise, so a
moved upstream tag cannot silently change what the image packages), installs
dependencies with pnpm, runs the production build, packages the `@deepseek-ai/dsh`
CLI with `pnpm pack`, and installs its production dependencies with npm.
2. **Runtime stage** (`node:24-trixie-slim`) copies `/opt/dsh` to `/app`, adds the
2. **Runtime stage** (`NODE_RUNTIME_IMAGE`) copies `/opt/dsh` to `/app`, adds the
CLI tools DSH agents commonly use (`git`, `ripgrep`, `jq`, `yq`, `curl`, `wget`,
`gh`, `tini`, `uv`), installs `bubblewrap` as DSH's preferred command-sandbox
runner, and starts `dsh web` through a small entrypoint.
@@ -36,6 +40,7 @@ To reproduce the equivalent by hand:
```bash
docker build \
--build-arg DSH_VERSION=dsh-v0.2.1-rc.5 \
--build-arg DSH_COMMIT=b830ec950faba0a46dda564be3e2d854afed94f1 \
--build-arg RELEASE_REVISION= \
--build-arg PNPM_VERSION=11.28.5 \
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
@@ -48,11 +53,17 @@ docker build \
| --- | --- | --- |
| `DSH_REPO` | `https://github.com/deepseek-ai/deepseek-harness.git` | Upstream DSH repository to clone. |
| `DSH_VERSION` | `DSH_VERSION` in [versions.env](../versions.env) | Upstream DSH git tag/ref to package. The Dockerfile records it as `io.sredevopsorg.dsh.version` and (for local builds) `org.opencontainers.image.version`; published builds set the version label to the full release identity. |
| `DSH_COMMIT` | `DSH_COMMIT` in [versions.env](../versions.env) | Full commit SHA `DSH_VERSION` must resolve to upstream. The build stage fails unless `git rev-parse HEAD` in the clone equals this value, which pins the source independently of a movable tag. |
| `RELEASE_REVISION` | `RELEASE_REVISION` in [versions.env](../versions.env) | Repository revision suffix (`rN`), recorded in `io.sredevopsorg.release.revision`. The release identity is `DSH_VERSION` plus `-RELEASE_REVISION` when set. See [versioning](versioning.md). |
| `PNPM_VERSION` | `PNPM_VERSION` in [versions.env](../versions.env) | pnpm version activated with corepack in both stages. |
| `VCS_REF` | `git rev-parse --short HEAD` (via `make image`) | Source revision recorded for traceability. |
| `NODE_BUILD_IMAGE` | `docker.io/node:24-trixie` | Build-stage base image. |
| `NODE_RUNTIME_IMAGE` | `docker.io/node:24-trixie-slim` | Runtime-stage base image. |
| `NODE_BUILD_IMAGE` | `docker.io/node:24-trixie@sha256:8c761eca2bf7eec965f946cf47b8090143cb41f45f2f8699af0cc0298d2d2d19` | Build-stage base image, pinned to the multi-arch index digest. |
| `NODE_RUNTIME_IMAGE` | `docker.io/node:24-trixie-slim@sha256:54f84b4cf8432b21aaffd5d3ce8d41e12cea0d10d33c09d2f3c9879d49d6b446` | Runtime-stage base image, pinned to the multi-arch index digest. |
Base images are pinned by digest because tags are mutable: `node:24-trixie` can be
re-pointed at a new build at any time. Bumping the tag and its digest is a
deliberate, reviewable change in [versions.env](../versions.env); environments that
mirror the image should also mirror the `node` digest they build from.
Never hard-code a version outside [versions.env](../versions.env):
`./scripts/check-versions.sh` (and `make check`) fails when a pin drifts.

42
docs/versioning.md
View File

@@ -61,14 +61,27 @@ Values from [versions.env](../versions.env) at the current identity:
| --- | --- | --- |
| Upstream DSH source | `DSH_REPO` | [versions.env](../versions.env) |
| DSH ref | `dsh-v0.2.1-rc.5` | `DSH_VERSION` |
| DSH commit | `b830ec950faba0a46dda564be3e2d854afed94f1` | `DSH_COMMIT`; the main image build fails unless the clone's `HEAD` matches |
| Repository revision | `(empty)` | `RELEASE_REVISION` |
| pnpm | `11.28.5` | `PNPM_VERSION` |
| Node build image | `docker.io/node:24-trixie@sha256:8c761eca2bf7eec965f946cf47b8090143cb41f45f2f8699af0cc0298d2d2d19` | `NODE_BUILD_IMAGE`, digest-pinned |
| Node runtime image | `docker.io/node:24-trixie-slim@sha256:54f84b4cf8432b21aaffd5d3ce8d41e12cea0d10d33c09d2f3c9879d49d6b446` | `NODE_RUNTIME_IMAGE`, digest-pinned |
| Gateway Python image | `docker.io/python:3.12-slim@sha256:aa943ce411423c7582288f18707ff66eddcbaeb510502d747a72aeb34f30d159` | `GATEWAY_PYTHON_IMAGE`, digest-pinned |
| agent-sandbox (CRDs) | `v1.0.9` | `AGENT_SANDBOX_VERSION`, used by the examples and CI dry-run |
| Kubernetes Gateway API | `v1.2.1` | `GATEWAY_API_VERSION`, used by the SSO exposure variant |
| oauth2-proxy | `quay.io/oauth2-proxy/oauth2-proxy:v7.15.5` | `OAUTH2_PROXY_IMAGE` |
| oauth2-proxy | `quay.io/oauth2-proxy/oauth2-proxy:v7.15.5@sha256:8498b0d0ef0a7b29686414000a08aee467f02d0299c9ed1e006a8f33fc017916` | `OAUTH2_PROXY_IMAGE`, digest-pinned and used by the SSO exposure variant |
| Main image | `ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox` | `GHCR_IMAGE` |
| Gateway image | `ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox/dsh-agents-as-a-service/gateway` | `GATEWAY_IMAGE`, nested below `GHCR_IMAGE` |
Every container base image is referenced as `name:tag@sha256:<index digest>`, not
by tag alone: `check-versions.sh` rejects a pin without a digest, so a re-pushed
tag cannot change what we build from. The example gateway's Python dependencies
are fully hash-pinned in
[requirements.txt](../examples/dsh-agents-as-a-service/gateway/requirements.txt)
and
[requirements-dev.txt](../examples/dsh-agents-as-a-service/gateway/requirements-dev.txt)
and installed with `pip install --require-hashes`.
Example component images are always nested below the main package:
`<GHCR_IMAGE>/<example>/<component>`. `check-versions.sh` rejects an unrelated
top-level package.
@@ -86,8 +99,15 @@ file to derive from it. Never edit the Dockerfile, manifests, Makefile or
workflows with a literal version.
- New upstream DSH ref: change `DSH_VERSION` and clear `RELEASE_REVISION`.
- Repository-only change at the same DSH ref: keep `DSH_VERSION`, set
`RELEASE_REVISION=r(N+1)`.
`set-version` resolves the new ref's commit with `git ls-remote` (peeled `^{}`
commit first, tag ref as fallback) and rewrites `DSH_COMMIT` in `versions.env`,
the Dockerfile ARG default and `.env.sample`; the build then verifies it.
- Repository-only change at the same DSH ref: keep `DSH_VERSION` and `DSH_COMMIT`,
set `RELEASE_REVISION=r(N+1)`.
- Base images (Node and Python) are bumped by hand: change the tag *and* the
`@sha256:` digest in `versions.env` together, then run `make check` and a build.
There is deliberately no automated digest refresh, because a silent base-image
change is exactly what digest pinning exists to prevent.
## Release checklist
@@ -98,7 +118,8 @@ workflows with a literal version.
the tag does not already exist, and previews the generated notes.
5. Create and push the annotated `dsh-*` release tag (or dispatch
`release.yaml` with an existing tag). The workflow validates that the tag
matches [versions.env](../versions.env).
matches [versions.env](../versions.env) and that the upstream ref still
resolves to `DSH_COMMIT` before building anything.
6. The workflow builds and pushes multi-arch images for the main image and the
example components (the identity tag plus `latest`), creates the GitHub
Release from the git-cliff notes, and opens a `chore(changelog)` PR that
@@ -142,8 +163,17 @@ make changelog # regenerate CHANGELOG.md locally
## Digest pinning
Release identities are mutable only in theory; deployments should still pin the
resolved digest for reproducibility and supply-chain safety:
Supply-chain inputs are pinned to immutable content, not to mutable tags:
- **Build inputs**: every container base image is referenced by multi-arch index
digest in [versions.env](../versions.env), and the example gateway's Python
dependencies are hash-pinned (`--require-hashes`).
- **Upstream source**: the Dockerfile clones `DSH_VERSION` and fails unless the
clone's `HEAD` equals `DSH_COMMIT`, so a moved upstream tag cannot change what
a rebuild packages.
Release identities are immutable in theory; deployments should still pin the
_resolved_ digest for reproducibility and supply-chain safety:
```bash
docker buildx imagetools inspect ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.5

215
examples/dsh-agents-as-a-service/README.md
View File

@@ -16,8 +16,9 @@ It ships in three layers:
1. **The resource layer** (steps 1–7): pure manifests + kubectl, so every
mechanism is visible.
2. **The service layer** (step 8): a small included **gateway** (`gateway/`) that
makes the platform browser-usable — a signup page/API, `HttpOnly` cookie
sessions, a `/u/{user}/**` proxy for HTTP **and WebSocket** that transparently
makes the platform browser-usable — a signup page/API, revocable server-side
**opaque-cookie sessions**, auth rate limiting and an optional signup invite
code, a `/u/{user}/**` proxy for HTTP **and WebSocket** that transparently
**wakes suspended instances on connect**, and an idle sweeper that suspends
them again. The gateway performs the DSH launch-token handshake server-side,
so the browser never sees the token.
@@ -65,12 +66,13 @@ What it demonstrates, feature by feature:
upstream instead of splitting `/v1/*` (API) from the dashboard.
- **Shared provider keys**: `DEEPSEEK_API_KEY` plus any keys in the optional
`dsh-provider-keys-extra` Secret are injected via `envFrom` into every
sandbox. There is no per-tenant key, dashboard basic-auth, session-cookie
secret, or API-server key.
sandbox. There is no per-tenant key, dashboard basic-auth, signed-session
secret (sessions are opaque server-side ids), or API-server key.
- **Launch-token handshake**: DSH gates `/` behind a token it prints only in its
logs. The gateway reads it, performs the handshake, and keeps the resulting
auth cookie server-side; the browser authenticates to the gateway with its own
cookie. See [How the DSH handshake works](#how-the-dsh-handshake-works).
opaque session cookie, never the token. See
[How the DSH handshake works](#how-the-dsh-handshake-works).
- **Host allow-list**: DSH rejects unknown `Host` headers. The gateway forwards
the browser's `Host`, so the public hostname must be added to
`DSH_TRUSTED_HOSTS` in the template.
@@ -285,6 +287,15 @@ kubectl -n dsh-demo port-forward svc/dsh-gateway 18080:8080
> The gateway binds `8080`; the local port-forward uses `18080` and never `3080`.
>
> **Cookie/TLS guard.** The base manifest serves plain HTTP through port-forward,
> so it sets `COOKIE_SECURE=false` **and** `ALLOW_INSECURE_COOKIES=1`. The gateway
> refuses to start when `COOKIE_SECURE=false` unless
> `ALLOW_INSECURE_COOKIES=1` explicitly acknowledges that the session cookie
> travels in clear text; that prevents an HTTPS deployment from silently serving
> an insecure cookie. Before exposing the gateway over TLS, set
> `COOKIE_SECURE=true` and remove the acknowledgement:
> `kubectl -n dsh-demo set env deploy/dsh-gateway COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-`.
>
> **Image pull policy.** `manifests/50-gateway.yaml` uses
> `imagePullPolicy: IfNotPresent` with the release identity, so a locally built
> image must be loaded into the cluster under exactly that name (kind:
@@ -317,25 +328,40 @@ kubectl -n dsh-demo port-forward svc/dsh-gateway 18080:8080
### Browser flow
Open <http://localhost:18080>. The landing page has a one-field signup form.
Enter `bob`, and the gateway:
Open <http://localhost:18080>. The landing page has a one-field signup form (plus
an invite-code field when `SIGNUP_INVITE_CODE` is configured). Enter `bob`, and
the gateway:
1. creates the `SandboxClaim` over the warm pool,
2. waits for it to become `Ready`,
3. sets an `HttpOnly`, `SameSite=Lax` cookie `dsh_gw_session=<user>:<token>` on
the browser,
4. redirects to `/u/bob/`, where the DSH Web UI loads through the gateway.
1. enforces the auth rate limit and, if configured, the signup invite code,
2. creates the `SandboxClaim` over the warm pool,
3. waits for it to become `Ready`,
4. mints a random, server-side session (the cookie value is an opaque `sid`, not
a user name and not a token), stores `sid -> (user, expiry)` in memory, and
sets the `HttpOnly`, `SameSite=Lax` cookie `dsh_gw_session=<sid>` on the
browser (`__Host-dsh_gw_session` when `COOKIE_SECURE=true`),
5. redirects to `/u/bob/`, where the DSH Web UI loads through the gateway.
From then on every request under `/u/bob/` (assets, API calls, SSE, WebSocket) is
proxied to bob's sandbox. The DSH launch token and DSH's own auth cookie stay in
the gateway; the browser only holds the gateway cookie.
the gateway; the browser only holds the opaque gateway session id. The cookie is
useless as a bearer token.
**Sessions are revocable and mostly short-lived in practice.** `POST /logout`,
deleting the user, `POST /recreate` and cookie expiry (default 7 days,
`COOKIE_MAX_AGE`) invalidate the server-side session. The session table lives in
gateway memory, so **restarting the gateway logs every browser out** — expected
for this single-replica demo; a production deployment needs a shared session
store and sticky routing or per-request re-authentication. With SSO enabled, the
gateway mints a session for the identity oauth2-proxy asserts instead of asking
for a token.
The same flow with `curl`, no JavaScript (the session cookie is the only
credential):
```bash
JAR=$(mktemp)
# signup (form POST, exactly what the landing page sends)
# signup (form POST, exactly what the landing page sends); add -d 'invite=...'
# when SIGNUP_INVITE_CODE is configured, or send X-Signup-Invite
curl -s -i -c "$JAR" -H 'Origin: http://localhost:18080' \
-d 'user=bob' http://localhost:18080/users
# proxy: bob's DSH Web through the gateway
@@ -344,13 +370,38 @@ curl -s -b "$JAR" http://localhost:18080/u/bob/ | head -5
curl -s -o /dev/null -w '%{http_code}\n' http://localhost:18080/u/bob/ # 401
# derived state: Ready / Waking / Suspended / ...
curl -s -b "$JAR" http://localhost:18080/users/bob
# delete bob (cascades sandbox + PVC)
# sign out (revokes the server-side session)
curl -s -b "$JAR" -H 'Origin: http://localhost:18080' -X POST http://localhost:18080/logout
# delete bob (cascades sandbox + PVC and revokes every session for bob)
curl -s -b "$JAR" -H 'Origin: http://localhost:18080' -X DELETE http://localhost:18080/users/bob
```
API clients that prefer bearer tokens can use `POST /users` with a JSON body
(`{"user":"bob"}`) and the returned one-time token, `POST /login` to exchange a
token for a cookie, and `Authorization: Bearer <token>` on subsequent calls.
token for a cookie, and `Authorization: Bearer <token>` on subsequent calls. A
cookie/session value is never accepted as a bearer token, and a token for one
user cannot read another user's path.
### Auth abuse controls and proxy limits
The gateway is a single process, so these limits are per replica. They stop
casual abuse and one tenant monopolizing the service; an internet-facing
deployment still needs ingress-level rate limiting and an audit trail.
| Setting | Default | Behavior |
| --- | --- | --- |
| `AUTH_RATE_LIMIT` / `AUTH_RATE_WINDOW` | `30` per `60` s | Throttles `/users` and `/login` per client (rightmost `X-Forwarded-For` when behind a proxy, else the TCP peer) and answers `429` + `Retry-After` before reading the body or touching the cluster. `0` disables it. The base manifest sets it explicitly. |
| `SIGNUP_INVITE_CODE` | unset | When set, `POST /users` requires the code as form field `invite` or header `X-Signup-Invite` (constant-time compare, `403` before user lookup). Put it in a Secret and restart the gateway to rotate. |
| `MAX_IN_FLIGHT_PER_USER` | `32` | Caps concurrent proxied requests + WebSockets per user; excess HTTP gets `503` + `Retry-After`, excess WebSocket upgrades close with `1011`. |
| `WS_MAX_MESSAGE_BYTES` | `16 MiB` | Largest WebSocket message received from DSH; oversized frames close the connection. Browser-to-DSH frames are bounded by uvicorn's own limit. HTTP/SSE streaming is not capped by this. |
| `MAX_BODY_MB` | `32` | Largest proxied HTTP request body (`413`). |
The gateway also strips client-supplied `Forwarded`, `X-Forwarded-User` and
`X-Auth-Request-*` headers (and `x-forwarded-*` / `x-auth-request-*` prefixes)
before proxying, so a tenant cannot inject identity or spoof hop headers into
DSH. `UPSTREAM_HEADER_ALLOWLIST` can replace that denylist with an explicit
comma-separated allowlist. See the [gateway README](gateway/README.md) for the
full configuration.
### Gateway routes
@@ -395,8 +446,11 @@ not cut off.
The upstream upgrade dials the sandbox **pod IP**, forwards no `Host`/`Origin`,
and performs the launch-token handshake under the same `pod-IP:3080` authority.
Missing or cross-origin auth closes with `4401`; upstream bridging failures close
with `1011`. See the [gateway README](gateway/README.md#proxy-failure-semantics)
for the full matrix.
with `1011`, as does exceeding `MAX_IN_FLIGHT_PER_USER` (per-user socket cap).
Messages received from DSH larger than `WS_MAX_MESSAGE_BYTES` (16 MiB by
default) close the connection instead of being buffered; browser-to-DSH frames
are bounded by uvicorn's own limit. See the
[gateway README](gateway/README.md#proxy-failure-semantics) for the full matrix.
### Path routing
@@ -420,17 +474,20 @@ not a stack. Details, commands and the switch procedure are in the
Both variants also ship NetworkPolicies: `ingress/61-networkpolicy.yaml` limits
gateway ingress to Traefik pods, and `gateway-api-sso/75-networkpolicies.yaml`
limits the gateway to oauth2-proxy and oauth2-proxy egress to DNS, public HTTPS
and the gateway. They are part of the variant; see the
[exposure README](manifests/exposure/README.md) and
[optional hardening](docs/optional-hardening.md).
and the gateway. They are part of the variant **and required for production**:
an exposure applied without its policy is demo-only. The sandbox side needs a
separate policy, and the gateway needs an egress policy; both are covered in
[optional hardening](docs/optional-hardening.md) (see the production checklist)
and summarized in the [exposure README](manifests/exposure/README.md).
### Option A — plain Ingress (gateway's own session auth)
1. Set the Ingress host/TLS (or drop the `tls:` block), and set
`DSH_TRUSTED_HOSTS` in `manifests/10-sandboxtemplate.yaml` to the same public
host; re-apply the template.
2. Set `COOKIE_SECURE=true` once the Ingress serves HTTPS:
`kubectl -n dsh-demo set env deploy/dsh-gateway COOKIE_SECURE=true`.
2. Set `COOKIE_SECURE=true` and drop the plain-HTTP acknowledgement once the
Ingress serves HTTPS:
`kubectl -n dsh-demo set env deploy/dsh-gateway COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-`.
3. `kubectl apply -f manifests/exposure/ingress/`; verify
`curl -sI https://<host>/healthz` returns `200`.
@@ -453,21 +510,37 @@ front of **oauth2-proxy**, which authenticates the browser against GitHub
2. Create a GitHub OAuth App whose homepage/callback match the public FQDN, and
create the oauth2-proxy and TLS Secrets from
`73-oauth2-proxy-secret.yaml.example` and `74-tls-secret.yaml.example`.
3. Replace `dsh-demo.example.com` and `your-github-org`, set
`DSH_TRUSTED_HOSTS` in the template, and enable SSO:
`kubectl -n dsh-demo set env deploy/dsh-gateway AUTH_PROXY_HEADER=X-Forwarded-User COOKIE_SECURE=true`.
4. `kubectl apply -f manifests/exposure/gateway-api-sso/`; verify the Gateway,
HTTPRoute, oauth2-proxy Deployment and endpoints as shown in the
3. Replace `dsh-demo.example.com` and `your-github-org` in the variant
manifests, set `DSH_TRUSTED_HOSTS` in the template, and enable SSO:
`kubectl -n dsh-demo set env deploy/dsh-gateway AUTH_PROXY_HEADER=X-Forwarded-User COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-`.
4. `kubectl apply -f manifests/exposure/gateway-api-sso/`; then run
[`scripts/preflight-sso.sh`](scripts/preflight-sso.sh) and fix everything it
reports **before** pointing real traffic at the Gateway. It verifies the
Secrets (including a valid oauth2-proxy cookie secret length), the gateway's
`AUTH_PROXY_HEADER`/`COOKIE_SECURE`, that the Gateway's TLS certificate
reference resolves, and that no `your-github-org`/`dsh-demo.example.com`
placeholder was left applied. Full verification steps are in the
[exposure README](manifests/exposure/README.md).
> **Exposure without the variant NetworkPolicy is demo-only.** The SSO
> variant's `75-networkpolicies.yaml` is not optional in practice: it is the
> only thing that makes oauth2-proxy the sole path to the gateway. Applying
> only the Gateway/HTTPRoute/oauth2-proxy objects leaves the trust boundary
> unenforced, so anyone who can reach `dsh-gateway:8080` can send
> `X-Forwarded-User` and impersonate the user they claim to be.
>
> **SSO trust boundary — read this before enabling it.** After SSO is on, the
> `dsh-gateway` Service must be reachable **only** through oauth2-proxy (it is
> ClusterIP). The variant's `75-networkpolicies.yaml` restricts pod traffic
> accordingly: the gateway accepts traffic only from oauth2-proxy pods,
> oauth2-proxy only from the Gateway implementation, and its egress only to DNS,
> public HTTPS and the gateway. Anyone who can reach `dsh-gateway:8080` directly
> can send `X-Forwarded-User: <victim>`, so never add a second path. To turn SSO
> off, remove the variable
> public HTTPS and the gateway. The gateway's own egress (Kubernetes API,
> sandbox pods) is not restricted by that file because the API-server endpoint is
> cluster-specific; add the
> [gateway egress policy](docs/optional-hardening.md#gateway-egress-required-for-production)
> for production. Anyone who can reach `dsh-gateway:8080` directly can send
> `X-Forwarded-User: <victim>`, so never add a second path. To turn SSO off,
> remove the variable
> (`kubectl -n dsh-demo set env deploy/dsh-gateway AUTH_PROXY_HEADER-`) and
> delete the SSO variant.
@@ -481,25 +554,36 @@ gateway remains. Full commands and verification are in the
## Optional hardening
The base walkthrough is deliberately portable. Five opt-in settings are
documented in [optional hardening](docs/optional-hardening.md):
The base walkthrough is deliberately portable, which means it is **not
hardened**: without the policies and gVisor it is demo-only, not a safe
multi-tenant platform. [Optional hardening](docs/optional-hardening.md) has the
settings and a production checklist. In short:
- **gVisor** — uncomment `runtimeClassName: gvisor` before applying, or patch an
existing install. Requires a `gvisor` RuntimeClass and capable nodes.
`kubectl port-forward` does not work under gVisor; the gateway path does.
Required for untrusted agents in production.
- **nodeSelector** — uncomment and replace the placeholder node label to pin
sandboxes and the gateway to a dedicated pool; add a toleration for tainted
pools.
- **Sandbox NetworkPolicy** —
`kubectl apply -f manifests/optional/network-policy/` allows only
gateway→sandbox on `3080` and sandbox→DNS/public `443`.
gateway→sandbox on `3080` and sandbox→DNS/public `443`. Required for
production; exposure without it is demo-only.
- **Gateway egress policy** — the gateway needs DNS, the Kubernetes API and
`3080` to sandbox pods. It is not shipped because the API-server endpoint is
cluster-specific; use the
[recipe](docs/optional-hardening.md#gateway-egress-required-for-production).
Required for production.
- **Provider keys** — create `dsh-provider-keys-extra` as shown in step 1.
- **Failed-Pod recovery** — uncomment `failedPodPolicy: Recreate` so the
controller replaces a `Failed` sandbox Pod (node eviction) instead of parking
the sandbox at `Finished=True`. Requires agent-sandbox v1.0.8+.
Apply each independently and re-run the walkthrough. `kubectl apply -f manifests/`
is non-recursive, so the optional manifests are not applied by the base step.
The **exposure variants' NetworkPolicies** are not optional in production
either; see step 9. Apply each setting independently and re-run the walkthrough.
`kubectl apply -f manifests/` is non-recursive, so the optional manifests are not
applied by the base step.
## Troubleshooting
@@ -508,9 +592,16 @@ is non-recursive, so the optional manifests are not applied by the base step.
| Pod `CrashLoopBackOff`, probes failing with `401` | Probe path is `/`; change readiness/liveness to `/favicon.svg` |
| Browser shows `401` from DSH through the gateway | Gateway handshake failed — check `kubectl -n dsh-demo logs deploy/dsh-gateway` and that RBAC grants `pods/log` |
| `502` / `upstream unavailable` | Sandbox pod is not Ready (wake timed out) or the Service DNS is stale; check `kubectl get sandbox`, `WAKE_TIMEOUT` |
| `503` with `Retry-After` | The launch token was not found in the pod logs yet; the gateway retries after a short delay. Check the pod logs and `DSH_LOG_TAIL_LINES` |
| `503` with `Retry-After` | Either the launch token was not found in the pod logs yet (the gateway retries after a short delay; check the pod logs and `DSH_LOG_TAIL_LINES`) or the user hit `MAX_IN_FLIGHT_PER_USER` (default 32 concurrent requests/WebSockets); reduce client concurrency or raise the limit deliberately |
| `429` with `Retry-After` on signup/login | The per-client auth rate limit (`AUTH_RATE_LIMIT`, default explicitly `30` per `AUTH_RATE_WINDOW=60` s) rejected the attempt before reading the body. Wait for the window, or raise the limit; behind a proxy make sure the real client IP is the rightmost `X-Forwarded-For` entry, not a shared NAT address |
| `403` on signup: invite required/invalid | `SIGNUP_INVITE_CODE` is configured. Send the code as form field `invite` or header `X-Signup-Invite`; the landing page shows the field when enabled. A wrong code is rejected before the user lookup |
| Gateway pod `CrashLoopBackOff` with `refusing to start with an insecure session cookie` | Intentional guard: plain HTTP is only allowed when explicitly acknowledged. For the local demo set `ALLOW_INSECURE_COOKIES=1` (the base manifest does); for HTTPS set `COOKIE_SECURE=true` and remove `ALLOW_INSECURE_COOKIES` |
| Every browser is logged out after a gateway rollout/restart | Expected: sessions are opaque ids in gateway memory and a restart drops them. Sign in again; a production deployment needs shared session state or per-request re-authentication. The DSH sandboxes and their data are unaffected |
| WebSocket closes with `1011` for a busy client | `MAX_IN_FLIGHT_PER_USER` (per-user connection cap) or an upstream/handshake failure closed the socket; check the gateway logs to tell them apart |
| WebSocket closes after a very large message from DSH | A message received from DSH exceeded `WS_MAX_MESSAGE_BYTES` (16 MiB), so the relay closed it instead of buffering; raise the limit only if DSH legitimately sends larger frames. Browser-to-DSH frames are bounded by uvicorn's own limit |
| `scripts/preflight-sso.sh` exits non-zero | It names the failing item: a missing Secret/key, a bad oauth2-proxy cookie-secret length, missing `AUTH_PROXY_HEADER`/`COOKIE_SECURE`, a dangling Gateway TLS reference, a leftover `your-github-org`/`dsh-demo.example.com` placeholder, or a second path to the gateway (an Ingress). Fix all items before exposing the SSO variant |
| `409` on signup | User already has a claim; use `POST /login` with the existing token or delete the user first |
| Cookie set but browser keeps getting `401` | `COOKIE_SECURE=true` while serving over plain HTTP (secure cookies are not sent); use HTTPS or set `false` for local demos |
| Cookie set but browser keeps getting `401` | `COOKIE_SECURE=true` while serving over plain HTTP (secure cookies are not sent); use HTTPS or set `false` **and** `ALLOW_INSECURE_COOKIES=1` for local demos |
| WebSocket closes immediately (`4401`) | Missing/expired gateway cookie, or the upgrade's `Origin` does not match `Host` (CSWSH check); DSH rejecting `Host` instead surfaces as close `1011` — add the host to `DSH_TRUSTED_HOSTS` |
| Console spams `[connection] connection lost, retry #N` | The upstream WS upgrade is being rejected, surfaced as close `1011`. The gateway dials the sandbox **pod IP**, forwards no `Host`/`Origin`, and handshakes under the same `pod-IP:3080` authority DSH binds its cookie to; an older gateway image, or a sandbox with an empty `status.podIPs`, hits this loop |
| `manifest.webmanifest` returns `401` in the console | Browsers fetch the Web App Manifest without credentials, so the gateway's session cookie is absent and the request fails closed. The UI is unaffected; only the PWA install prompt is lost |
@@ -539,9 +630,19 @@ is non-recursive, so the optional manifests are not applied by the base step.
Deliberately **not** production code:
- **Single replica.** Idle clocks, in-flight tracking, and the handshake cache
are in memory. Running two gateway replicas would double-suspend or lose wake
tracking; scale up only after moving that state to a shared store.
- **Single replica.** Idle clocks, in-flight tracking, the handshake cache and
the browser session table are in memory. Running two gateway replicas would
double-suspend or lose wake tracking, and sessions would land on the wrong
replica; scale up only after moving that state to a shared store.
- **Sessions die with the process.** Browser sessions are opaque ids held in
gateway memory (revocable on logout/delete/recreate, but not durable). A
gateway restart logs every browser out. There is no SSO refresh-token or
password store; identity is re-established by oauth2-proxy on the next
request.
- **Abuse controls are per-process and coarse.** `AUTH_RATE_LIMIT`, the signup
invite code, `MAX_IN_FLIGHT_PER_USER` and `WS_MAX_MESSAGE_BYTES` limit casual
abuse, but they are not a substitute for ingress rate limiting, WAF/quota, or
an audit log, and the in-memory rate buckets reset on restart.
- **Log scraping.** The launch token is recovered from pod logs. Log rotation, a
restarted pod, or a changed DSH log format can break the handshake until the
next `401` refresh.
@@ -551,12 +652,13 @@ Deliberately **not** production code:
but the platform does not manage that. Real multi-tenancy needs per-tenant
credentials or an egress proxy.
- **No TLS in the gateway.** Terminate TLS at an Ingress or Gateway and set
`COOKIE_SECURE=true`.
`COOKIE_SECURE=true`, then remove `ALLOW_INSECURE_COOKIES`. The base manifest
runs plain HTTP for the port-forward demo and is demo-only.
- **SSO trust is hop-based.** `AUTH_PROXY_HEADER` trusts whoever sets the header.
The SSO variant ships `75-networkpolicies.yaml`, which restricts the gateway
to oauth2-proxy and oauth2-proxy egress to the gateway/DNS/public HTTPS; a
custom exposure without that policy has no signature, mTLS or network
enforcement.
enforcement. Exposure without the policy is demo-only.
- **DSH's inner file sandbox is off here.** The template sets
`DSH_PERMISSION_MODE=danger-full-access` because neither backend of DSH's own
command sandbox is usable in this pod: `bubblewrap` is installed in the image
@@ -567,21 +669,24 @@ Deliberately **not** production code:
pod remains the boundary — non-root,
capabilities dropped, `seccompProfile: RuntimeDefault`, its own PVC and Service
— but an agent command can now read its own `DSH_HOME` (including
`.credentials.yaml`) and write anywhere in the container. On hosts where a
backend works, set `workspace-write` (or `read-only`) and verify before relying
on the inner policy.
- **Isolation is opt-in.** gVisor, the sandbox NetworkPolicy and the exposure
policies are opt-in; without them, sandboxes share the host kernel and can
reach the cluster network. The base example is a demo, not a hardened
multi-tenant platform.
`.credentials.yaml`) and write anywhere in the container. Run sandboxes under
gVisor in production. On hosts where a backend works, set `workspace-write`
(or `read-only`) and verify before relying on the inner policy.
- **Isolation is opt-in.** gVisor, the sandbox NetworkPolicy, the gateway egress
policy and the exposure policies are opt-in; without them, sandboxes share the
host kernel and can reach the cluster network. The base example is a demo, not
a hardened multi-tenant platform. See the
[production checklist](docs/optional-hardening.md#6-production-checklist).
- **OAuth org is the only authorization.** Any GitHub user in the configured
organization gets an auto-provisioned sandbox; there is no per-user allow-list,
quota, or admin API.
quota, or admin API. `SIGNUP_INVITE_CODE` gates the gateway's own signup form,
not SSO auto-provisioning.
- **In-memory user directory.** Users are claims + annotations; there is no
database, password reset, or admin API. Tokens are stored only as SHA-256
hashes in claim annotations.
- **Bearer tokens in URLs.** Avoid putting the signup token in a URL/log; the
browser flow uses the `HttpOnly` cookie instead.
browser flow uses the `HttpOnly` cookie instead. The cookie no longer contains
the token, and a cookie value is never accepted as a bearer token.
## Cleanup
@@ -603,6 +708,8 @@ delete the Secrets with the namespace and revoke the OAuth App in GitHub.
- [Architecture](docs/architecture.md) — container/sequence diagrams, ADRs,
security and RBAC, failure modes, evolution path.
- [Exposure README](manifests/exposure/README.md) — the two variants in detail.
- [SSO preflight](scripts/preflight-sso.sh) — read-only checks for the SSO
variant (Secrets, gateway env, TLS reference, placeholders, second paths).
- [Optional hardening](docs/optional-hardening.md) — gVisor, nodeSelector,
NetworkPolicy, extra provider keys and failed-Pod recovery, with verification
and rollback.
NetworkPolicy, extra provider keys and failed-Pod recovery, with verification,
the production checklist and rollback.

134
examples/dsh-agents-as-a-service/docs/architecture.md
View File

@@ -72,8 +72,10 @@ Each variant also ships a NetworkPolicy: `ingress/61-networkpolicy.yaml` limits
to Traefik pods, and `gateway-api-sso/75-networkpolicies.yaml` limits the gateway to
oauth2-proxy, oauth2-proxy ingress to the Gateway implementation, and oauth2-proxy egress to
DNS/public HTTPS/the gateway. The SSO policy turns the trust boundary into pod-traffic
enforcement; the caveats (Traefik label, `hostNetwork`, kubelet probes) are in
[optional hardening](optional-hardening.md).
enforcement. These policies are **required for production**: exposing the gateway by applying
only the Gateway/HTTPRoute/Ingress objects is demo-only. The sandbox-side policy and the
gateway's own egress policy are separate (the API-server endpoint is cluster-specific); see
[optional hardening](optional-hardening.md#gateway-egress-required-for-production).
## Flows
@@ -87,20 +89,22 @@ sequenceDiagram
participant K as Kubernetes API
participant C as agent-sandbox controller
B->>G: POST /users (form: user=bob, Origin)
G->>G: validate user name and body size
B->>G: POST /users (form: user=bob, invite, Origin)
G->>G: rate limit, invite check, validate user name and body size
G->>K: create SandboxClaim dsh-bob (warmPoolRef)
K->>C: reconcile claim
C-->>K: adopt spare, Ready, status.sandbox.name = dsh-pool-xxxx
G->>K: read claim (status) + sandbox + pod
G-->>B: 303 Location /u/bob/ + Set-Cookie dsh_gw_session
G->>G: mint opaque sid, store sid -> (user, expiry)
G-->>B: 303 Location /u/bob/ + Set-Cookie dsh_gw_session=<sid>
Note over G: JSON clients instead get 201 (Ready),<br/>202 (accepted, still waking), or 409 (conflict)
```
Form signup waits up to `WAKE_TIMEOUT` for the claim to become `Ready`, then redirects; the
first proxied request also waits for `Ready` and performs the DSH handshake (below). A `409`
means the user already has a claim, in which case `POST /login` (bearer token) or the existing
cookie is the right path.
cookie is the right path. The cookie value is an opaque session id: it is meaningless outside
this gateway process and is never accepted as a bearer token.
### First request / proxy (with handshake)
@@ -112,8 +116,8 @@ sequenceDiagram
participant K as Kubernetes API
participant D as DSH Web pod
B->>G: GET /u/bob/ (Cookie dsh_gw_session)
G->>G: authenticate cookie, user = bob
B->>G: GET /u/bob/ (Cookie dsh_gw_session=<sid>)
G->>G: look up sid -> session (valid, not expired), user = bob
G->>K: resolve claim -> sandbox -> pod
alt sandbox Suspended
G->>K: patch operatingMode=Running (wake)
@@ -140,6 +144,32 @@ Inside `proxy_http` the order is auth, in-flight slot, sandbox resolution/wake,
body read, upstream send. An upload aborted mid-body may therefore already have paid for a wake
and a handshake, but it never reaches DSH.
### Browser sessions
The browser never holds a DSH credential. On signup/login the gateway mints a random session id
(`secrets.token_urlsafe(32)`), stores `sid -> Session(user, expires_at)` in memory, and sets it
as the `HttpOnly`, `SameSite=Lax` cookie `dsh_gw_session` (`__Host-dsh_gw_session` when
`COOKIE_SECURE=true`). Every browser request is resolved through that table; the cookie value is
not a bearer token and cannot be replayed against another user's path.
Sessions end on explicit logout, user delete, `/recreate` (all sessions for that user) and
expiry (`COOKIE_MAX_AGE`, default 7 days). Because the table is process memory, a gateway
restart logs every browser out; scaling beyond one replica therefore requires a shared session
store (or re-authentication on every request). The sweeper prunes expired entries and sessions
for users whose claim no longer exists.
### Auth abuse controls and proxy limits
Unauthenticated endpoints are throttled before the body is read: `AUTH_RATE_LIMIT` attempts per
client per `AUTH_RATE_WINDOW` (defaults 30/60s; keyed on the rightmost `X-Forwarded-For` entry
when present, else the peer address), answered with `429` + `Retry-After`. When
`SIGNUP_INVITE_CODE` is set, signup additionally requires a constant-time-matched code (form
field or `X-Signup-Invite`) and answers `403` otherwise. Proxied traffic is capped per user
(`MAX_IN_FLIGHT_PER_USER`, default 32 concurrent HTTP requests + WebSockets: `503` for HTTP,
close `1011` for WebSocket) and per WebSocket message received from DSH
(`WS_MAX_MESSAGE_BYTES`, 16 MiB). These are in-process guards for a single replica, not an ingress
rate limiter or quota system.
### Wake-on-connect and idle suspend
```mermaid
@@ -220,6 +250,14 @@ sequenceDiagram
end
```
An upgrade with a missing/expired session, or one that would exceed
`MAX_IN_FLIGHT_PER_USER`, is accepted and then closed: `4401` for authentication, `1011` for
the per-user socket cap. Messages received from DSH above `WS_MAX_MESSAGE_BYTES` (16 MiB) close
the connection instead of being buffered; browser-to-DSH frames are bounded by uvicorn's own
limit. Client-supplied `Forwarded`, `X-Forwarded-User`,
`X-Auth-Request-*` and `x-forwarded-*`/`x-auth-request-*` headers are stripped on the HTTP and
WebSocket paths; `UPSTREAM_HEADER_ALLOWLIST` replaces that denylist with an explicit allowlist.
## Decisions and trade-offs
### ADR-1: Starlette ASGI instead of a sync Flask proxy
@@ -279,14 +317,21 @@ sequenceDiagram
(`kubectl annotate sandboxclaim ...`) to keep being swept. Fine for a reference pattern, not
for a product.
### ADR-6: Gateway cookie session instead of bearer tokens in the browser
### ADR-6: Opaque server-side session cookie instead of bearer tokens in the browser
- **Decision:** browsers get `dsh_gw_session=<user>:<token>`, `HttpOnly`, `SameSite=Lax`
(`Secure` when enabled); API clients may still use `Authorization: Bearer`.
- **Why:** tokens never appear in URLs or JS-readable storage, and `SameSite=Lax` plus a strict
`Origin`/`Referer` check on cookie-authenticated mutations blocks basic CSRF.
- **Cost:** cookie-based auth needs TLS and correct `COOKIE_SECURE`; there is no signed-cookie
layer, so the gateway trusts the token hash comparison for validation.
- **Decision:** browsers get `dsh_gw_session=<sid>`, `HttpOnly`, `SameSite=Lax` (`Secure` when
enabled); the sid maps to `Session(user, expires_at)` in gateway memory. API clients may still
use `Authorization: Bearer`. The cookie name is `__Host-dsh_gw_session` when the Secure flag
is on and no explicit `COOKIE_NAME` is set.
- **Why:** tokens never appear in URLs or JS-readable storage, the cookie value is useless if
leaked outside the gateway session table, sessions can be revoked server-side (logout,
user delete, recreate, expiry), and a cookie cannot be reused as a bearer token for another
user's path. `SameSite=Lax` plus a strict `Origin`/`Referer` check on cookie-authenticated
mutations blocks basic CSRF.
- **Cost:** stateful auth in a single process: a restart logs everyone out and horizontal
scaling needs a shared session store. Cookie-based auth also needs TLS and the correct
`COOKIE_SECURE` setting; the gateway refuses to start with `COOKIE_SECURE=false` unless
`ALLOW_INSECURE_COOKIES=1` is set for a local HTTP demo.
## Security and RBAC
@@ -305,33 +350,52 @@ Notes and limits:
permission is scoped to one namespace, but anyone who can read gateway memory or logs can
impersonate users. Prefer the upstream token API when available.
- **No cluster-wide access, no secret reads.** The gateway does not read `dsh-provider-keys`.
- **Browser-facing hardening:** `HttpOnly`/`SameSite=Lax`/optional `Secure` cookies,
timing-safe SHA-256 token comparison, strict same-origin check on cookie-auth mutations,
request-body limit (`MAX_BODY_MB`), hop-by-hop header stripping, upstream `Set-Cookie`
dropped, DSH `Location` headers rewritten under `/u/<user>/`.
(It does read the optional signup invite code if it is mounted as an env var.)
- **Browser-facing hardening:** opaque server-side sessions instead of token-bearing cookies,
`HttpOnly`/`SameSite=Lax`/optional `Secure` (+ `__Host-` prefix when secure),
timing-safe SHA-256 bearer-token comparison, timing-safe invite-code comparison, strict
same-origin check on cookie-auth mutations, request-body limit (`MAX_BODY_MB`), hop-by-hop
header stripping, client identity/hop headers (`Forwarded`, `X-Forwarded-User`,
`X-Auth-Request-*`, `x-forwarded-*`/`x-auth-request-*`) stripped on HTTP and WebSocket,
upstream `Set-Cookie` dropped, DSH `Location` headers rewritten under `/u/<user>/`.
- **Session lifecycle:** login/signup mint an opaque sid; logout and user delete revoke it;
`/recreate` revokes every session for that user; expiry (`COOKIE_MAX_AGE`, default 7 days) is
enforced on lookup. The table is in memory, so a restart logs browsers out and a second
replica needs shared state.
- **Abuse controls (per replica):** `AUTH_RATE_LIMIT`/`AUTH_RATE_WINDOW` throttle
unauthenticated signup/login per client with `429` + `Retry-After`; `SIGNUP_INVITE_CODE`
gates signup; `MAX_IN_FLIGHT_PER_USER` caps concurrent proxied requests + WebSockets per user;
`WS_MAX_MESSAGE_BYTES` caps messages received from DSH; browser-to-DSH frames are bounded by
uvicorn. These are in-process guards, not an ingress rate limiter or audit log.
- **Host allow-list:** DSH rejects unknown `Host` headers. The gateway forwards the browser's
`Host`, so every public hostname must be in `DSH_TRUSTED_HOSTS`; `localhost`/`127.0.0.1` are
always trusted.
- **TLS is external.** The gateway serves plain HTTP; terminate TLS at the Ingress and set
`COOKIE_SECURE=true`. Without TLS, session cookies are replayable on the network.
`COOKIE_SECURE=true`. The base demo sets `COOKIE_SECURE=false` with
`ALLOW_INSECURE_COOKIES=1` for the port-forward walkthrough; without TLS, session cookies are
replayable on the network and that combination must never be exposed.
- **Tenant isolation** is pod + PVC separation; all tenants share one namespace, one DSH binary,
and the shared provider keys (`dsh-provider-keys` plus the optional
`dsh-provider-keys-extra`). `manifests/optional/network-policy/` adds an opt-in sandbox
NetworkPolicy, and gVisor/`nodeSelector` are opt-in PodSpec fields. Add per-tenant
credentials before calling it multi-tenant-grade.
`dsh-provider-keys-extra`). The base example is demo-only: production requires gVisor for the
sandboxes, the sandbox NetworkPolicy, the gateway egress policy and the exposure NetworkPolicy,
plus per-tenant credentials or an egress proxy. See the
[production checklist](optional-hardening.md#6-production-checklist).
- **DSH's inner file sandbox is disabled by deployment override.** The template sets
`DSH_PERMISSION_MODE=danger-full-access` because the in-container backends are unusable here:
`bubblewrap` ships in the image but is unsetuid and cannot create an unprivileged user namespace
under the pod's seccomp/capability surface, and no Landlock runner is available; the confined
executor would otherwise fail closed on every command. The pod remains the boundary, so an
agent can read its own `DSH_HOME`. Hosts with a working backend can keep a confined mode.
agent can read its own `DSH_HOME`. Hosts with a working backend can keep a confined mode;
multi-tenant deployments must put the pod in gVisor and a NetworkPolicy.
- **Optional hardening is opt-in.** gVisor, node selection, the sandbox/exposure NetworkPolicies,
extra provider keys and failed-Pod recovery are documented in
[optional hardening](optional-hardening.md). The base manifests stay portable (no gVisor
RuntimeClass, no active node labels, no NetworkPolicy, no extra Secret required, and the
controller's default `failedPodPolicy: Retain`).
- **No gateway rate limiting.** Signup can create claims until the cluster or warm pool is
exhausted; put limits at the ingress for real deployments.
- **SSO is trust-on-hop.** `AUTH_PROXY_HEADER` makes the gateway trust the proxy-asserted
identity; the SSO variant must remain the only path to the Service, and its NetworkPolicy is
required, not optional. Run [`scripts/preflight-sso.sh`](../scripts/preflight-sso.sh) after
applying it.
## Failure modes
@@ -340,7 +404,14 @@ Notes and limits:
| DSH log line changed or rotated away | Proxy returns `503` (missing launch token) | Fix the parser or restart the pod so the token line is re-emitted |
| Probe on `/` | Pod `CrashLoopBackOff`, `401` in probe logs | Use `/favicon.svg` for readiness/liveness (shipped templates do) |
| Warm pool exhausted | JSON signup returns `202` (state `Waking`); form signup is slow | Increase `replicas` in the warm pool |
| Gateway restart | Idle sandboxes stop suspending; handshake cache lost | Rebuilt lazily on next request; consider a shared store for production |
| Gateway restart | Idle sandboxes stop suspending; handshake cache and browser sessions lost | Idle clocks and handshake cache rebuild lazily; sessions do not, so browsers must sign in again after a rollout. Use a shared session store for production |
| Gateway configured `COOKIE_SECURE=false` without `ALLOW_INSECURE_COOKIES=1` | Gateway pod exits at startup / `CrashLoopBackOff` | Intentional guard against an insecure cookie on an HTTPS deployment; set `ALLOW_INSECURE_COOKIES=1` only for the local plain-HTTP demo, or set `COOKIE_SECURE=true` |
| Auth rate limit hit | `429` + `Retry-After` on signup/login | Per-client in-memory bucket (`AUTH_RATE_LIMIT`/`AUTH_RATE_WINDOW`); wait for the window or tune it. Behind a proxy, keying uses the rightmost `X-Forwarded-For` entry |
| Signup invite missing/wrong | `403` before the claim is created | `SIGNUP_INVITE_CODE` is set; send the `invite` form field or `X-Signup-Invite` header |
| Per-user concurrency cap hit | HTTP `503` + `Retry-After`, or WebSocket close `1011` | `MAX_IN_FLIGHT_PER_USER` protects the single gateway from one busy tenant; reduce concurrency or raise the cap deliberately. These are per replica |
| Oversized WebSocket message from DSH | Connection closes | `WS_MAX_MESSAGE_BYTES` (16 MiB) is enforced on upstream messages; raise it only if DSH legitimately needs larger frames |
| Session expiry / revocation | `401` (HTTP) or close `4401` (WebSocket) | Expected after logout, user delete, `/recreate`, `COOKIE_MAX_AGE` expiry or a gateway restart; sign in again |
| SSO variant applied without its NetworkPolicy (or with an Ingress) | Gateway accepts spoofed `X-Forwarded-User` | Demo-only configuration; keep `75-networkpolicies.yaml`, remove the second path, and run `scripts/preflight-sso.sh` |
| Browser aborts a request (tab reload, back navigation) during a slow wake or handshake | No response reaches the client; nothing appears in the access log | If the abort lands before the response starts, the body read raises `ClientDisconnect` and the app answers `499` (`CLIENT_CLOSED_REQUEST`), which cannot reach the departed peer. Nothing reached DSH, no ERROR traceback is logged, and the in-flight slot is released. A request already streaming ends with its upstream status. |
| DSH auth cookie expired mid-session | Proxy `401` once, then recovers | Gateway refreshes the handshake once |
| Upstream pod name changed after resume | Cached cookie keyed by old pod UID is useless | Cache is invalidated on suspend; next request re-handshakes |
@@ -372,10 +443,11 @@ single-retry refresh are the supported workaround.
## Evolution path
1. Replace log scraping with the upstream token API (above), deleting the `pods/log` permission.
2. Move idle clocks, in-flight tracking, and handshake cache to a shared store; then scale the
gateway horizontally (sticky routing not required if state is shared).
2. Move idle clocks, in-flight tracking, handshake cache **and sessions** to a shared store; then
scale the gateway horizontally (sticky routing not required if state is shared).
3. Add per-tenant API keys or an egress proxy so a single key is not shared by all sandboxes.
4. Add NetworkPolicies (gateway → sandboxes only), TLS everywhere, rate limits, and audit
logging at the ingress.
4. Add NetworkPolicies (gateway → sandboxes only, plus the documented gateway egress policy),
gVisor for sandbox pods, TLS everywhere, ingress-level rate limits/quota and audit logging.
The gateway's in-process limits are not a substitute.
5. If DSH grows absolute URLs or per-user cookies that cannot be rewritten, switch to
per-user subdomains with wildcard DNS/TLS.

124
examples/dsh-agents-as-a-service/docs/optional-hardening.md
View File

@@ -2,14 +2,26 @@
The base example is deliberately portable: it runs on kind and GKE without a
gVisor runtime, a tainted node pool or a NetworkPolicy-capable CNI, and it
boots with only the required `dsh-provider-keys` Secret. This document covers
four **opt-in** controls plus failed-Pod recovery:
boots with only the required `dsh-provider-keys` Secret. That portability is
exactly why the base manifests are **demo-only**. This document covers the
controls you add for production and the steps to enable or roll them back.
**Production is not the base walkthrough.** At minimum, a production deployment
needs: gVisor for the sandbox pods (or another kernel-isolation boundary), the
sandbox NetworkPolicy, the gateway egress policy, the exposure variant's
NetworkPolicy, TLS with `COOKIE_SECURE=true` (and no
`ALLOW_INSECURE_COOKIES`), SSO (or an equivalent authenticating proxy) as the
only path to the gateway, and per-tenant provider credentials/egress. The
checklist at the end summarizes this; without the network and kernel layers the
example is not tenant-isolated.
This document covers four **opt-in** controls plus failed-Pod recovery:
| Setting | Where it lives | What it changes |
| --- | --- | --- |
| gVisor | commented `runtimeClassName` in the SandboxTemplates and Deployments | puts DSH and/or the gateway in a userspace kernel |
| gVisor | commented `runtimeClassName` in the SandboxTemplates and Deployments | puts DSH and/or the gateway in a userspace kernel; required for untrusted agents in production |
| nodeSelector | commented `nodeSelector` in the same files | pins pods to a chosen node pool |
| NetworkPolicy | `manifests/optional/network-policy/`, `manifests/exposure/*/` | limits sandbox, gateway and proxy connectivity |
| NetworkPolicy | `manifests/optional/network-policy/`, `manifests/exposure/*/`, plus the gateway egress recipe | limits sandbox, gateway and proxy connectivity; the sandbox and gateway-egress policies are required for production |
| Provider keys | optional `dsh-provider-keys-extra` Secret | injects extra provider API keys into every sandbox |
| Failed-Pod recovery | commented `failedPodPolicy` in the SandboxTemplates | replaces a Failed sandbox Pod instead of parking the sandbox |
@@ -234,10 +246,17 @@ kubectl -n dsh-demo patch sandboxtemplate dsh-web --type merge \
The base example does **not** manage NetworkPolicy: the SandboxTemplate is
`networkPolicyManagement: Unmanaged`, and the gateway has no policy. That keeps
the walkthrough portable and independent of the sandbox-router; the opt-in
policies below are standard Kubernetes `NetworkPolicy` objects.
the walkthrough portable and independent of the sandbox-router. **For
production this is not optional**: without the sandbox policy every tenant can
reach every other tenant and cluster-internal endpoints, and without a gateway
egress policy the gateway can talk to anything the cluster network allows.
Apply both. The policies below are standard Kubernetes `NetworkPolicy` objects.
### Optional sandbox policy
> **Demo vs production.** An installation that skips the sandbox NetworkPolicy
> and the gateway egress policy is a demo. Exposure (Ingress/Gateway API) adds
> ingress protection only; it does not isolate tenants from each other.
### Sandbox policy (required for production)
[`manifests/optional/network-policy/10-sandbox-networkpolicy.yaml`](../manifests/optional/network-policy/10-sandbox-networkpolicy.yaml)
selects every Sandbox pod through the agent-sandbox tracking label
@@ -301,10 +320,10 @@ Caveats:
- `kubectl port-forward` still works for debugging (the connection is local to
the pod), except under gVisor.
### Exposure-variant policies
### Exposure-variant policies (required, part of the variant)
Each exposure variant ships its own policy. They are part of the variant, not a
separate opt-in step:
Each exposure variant ships its own policy. They are part of the variant and
**required for production**; an exposure without its policy is demo-only:
- [`manifests/exposure/ingress/61-networkpolicy.yaml`](../manifests/exposure/ingress/61-networkpolicy.yaml)
allows only Traefik pods (`app.kubernetes.io/name: traefik`) to reach the
@@ -317,7 +336,10 @@ separate opt-in step:
The SSO policy is what turns the trust boundary in the
[exposure README](../manifests/exposure/README.md) into network enforcement.
The `dsh-gateway` Service must still stay ClusterIP; do not apply both exposure
variants.
variants. After applying the SSO variant, run
[`scripts/preflight-sso.sh`](../scripts/preflight-sso.sh): it checks the
gateway's SSO env, the Secrets, the TLS reference, leftover placeholders and
accidental second paths (such as an Ingress).
Verify the negative case from a sandbox pod (it is not the proxy):
@@ -332,11 +354,12 @@ different pod label, replace the `podSelector` with the correct selector or an
`ipBlock`. As with the sandbox policy, kubelet probes can be affected on some
CNIs.
### Gateway egress (documented, not shipped)
### Gateway egress (required for production)
The gateway needs DNS, the Kubernetes API, and TCP `3080` to sandbox pods. The
API server endpoint is cluster-specific (Service CIDR or node IP/port), so an
egress policy is **not** shipped. If your cluster has a stable API-server CIDR
The gateway needs DNS, the Kubernetes API, and TCP `3080` to sandbox pods. An
egress policy is **required for production** but is not shipped, because the API
server endpoint is cluster-specific (Service CIDR or node IP/port) and a wrong
value silently breaks signup/proxy. If your cluster has a stable API-server CIDR
and port, add an egress policy like this (replace the placeholders; verify the
port, commonly `443` or `6443`):
@@ -597,7 +620,63 @@ done
Warm-pool spares pick the default up as they are claimed and replenished, or
delete them as in step 2 above.
## 6. Verification summary
## 6. Production checklist
The base walkthrough is a demo. A production install must satisfy every item
below; the sections above give the commands and caveats.
**Isolation and network**
- [ ] Sandbox pods run under gVisor (or an equivalent kernel-isolation
boundary) because DSH runs with `DSH_PERMISSION_MODE=danger-full-access`
in this image; the pod is the boundary. (Section 1.)
- [ ] The sandbox NetworkPolicy is applied and verified: no tenant→tenant,
no tenant→cluster-Service/API, DNS + public HTTPS only. (Section 3.)
- [ ] The gateway egress policy is applied with the cluster's real API-server
CIDR/port, scoped to DNS, the API server and sandbox pods on `3080`.
(Section 3, "Gateway egress".)
- [ ] The exposure variant's NetworkPolicy is applied. An exposure without it is
demo-only. For SSO, the gateway is reachable **only** through
oauth2-proxy; there is no second Ingress/Gateway/port-forward path.
- [ ] Nodes/CNI enforce NetworkPolicy (verify the negative cases, not just the
positive path).
- [ ] The gateway and (optionally) oauth2-proxy run with `runAsNonRoot`,
dropped capabilities, `RuntimeDefault` seccomp and a read-only root
filesystem, as shipped.
**Authentication, TLS and abuse controls**
- [ ] TLS terminates in front of the gateway, `COOKIE_SECURE=true`, and
`ALLOW_INSECURE_COOKIES` is **not** set. The gateway refuses to start with
an insecure cookie unless explicitly acknowledged.
- [ ] SSO (or an equivalent authenticating proxy) is enabled with
`AUTH_PROXY_HEADER=X-Forwarded-User`, and
[`scripts/preflight-sso.sh`](../scripts/preflight-sso.sh) exits `0`.
- [ ] No `your-github-org` / `dsh-demo.example.com` placeholder remains.
- [ ] `SIGNUP_INVITE_CODE` is set from a Secret if the gateway's own signup form
is reachable, and `AUTH_RATE_LIMIT` is set deliberately (`30`/60s shipped).
Add ingress-level rate limiting/quota too; the gateway's buckets are
in-process and reset on restart.
- [ ] Session lifetime (`COOKIE_MAX_AGE`) and the logout/delete revocation
behavior are acceptable; account for gateway restarts logging users out.
- [ ] `MAX_IN_FLIGHT_PER_USER` and `WS_MAX_MESSAGE_BYTES` fit the workload.
**Credentials, platform and operations**
- [ ] Provider keys are per-tenant (or routed through an egress proxy) instead
of the shared `dsh-provider-keys` Secrets; any shared key is treated as
visible to every tenant.
- [ ] RBAC stays namespace-scoped; `pods/log` is treated as sensitive because
it contains launch tokens and model output. Plan the upstream token API
migration.
- [ ] Images are digest-pinned and dependency installs are hash-locked.
- [ ] Gateway replicas stay at `1` (in-memory sessions/idle state) or the state
is externalized.
- [ ] `failedPodPolicy: Recreate` is set if node eviction/spot reclaim is
expected.
- [ ] Audit logging, alerting and backups of the PVCs are in place.
## 7. Verification summary
Required locally for any change to this example:
@@ -605,6 +684,7 @@ Required locally for any change to this example:
make check # versions, example layout, manifest YAML shape
yamllint .
make test # only when the gateway code changed
shellcheck examples/dsh-agents-as-a-service/scripts/*.sh
```
CI additionally parses every manifest and runs `kubectl apply
@@ -618,17 +698,25 @@ Manual checks, on a cluster with the relevant CNI/runtime:
gateway proxy works, port-forward limitation acknowledged.
- nodeSelector: pods land on the intended nodes; no `Pending` events.
- Sandbox policy: the allow/deny matrix above.
- Gateway egress policy: signup and proxy still work; the gateway cannot reach
an arbitrary cluster Service or the metadata server.
- Exposure policies: browser path works; a non-proxy pod cannot reach the
gateway; probes stay healthy.
gateway; probes stay healthy. `scripts/preflight-sso.sh` exits `0`.
- Cookie/TLS guard: with `COOKIE_SECURE=false` and no
`ALLOW_INSECURE_COOKIES` the gateway refuses to start; with TLS it starts and
the cookie is `Secure` (`__Host-` prefixed).
- Auth abuse controls: `429` + `Retry-After` after the configured limit, `403`
on a wrong invite code, and revocations on logout/delete/recreate.
- Provider keys: set/unset output (never the values) and the refresh procedure.
- Failed-Pod recovery: `spec.failedPodPolicy` on the Sandbox, and the drill
above replaces a `Failed` Pod while keeping the claim and PVC.
## 7. Rollback summary
## 8. Rollback summary
| Layer | Rollback |
| --- | --- |
| Sandbox policy | `kubectl delete -f manifests/optional/network-policy/` |
| Gateway egress policy | `kubectl delete networkpolicy dsh-gateway-egress -n dsh-demo` (do not leave it half-configured) |
| Exposure policy | delete the exposure variant directory |
| gVisor | patch `runtimeClassName: null`, recreate pods |
| nodeSelector | patch `nodeSelector: null`, recreate pods |

67
examples/dsh-agents-as-a-service/gateway/Dockerfile
View File

@@ -1,16 +1,73 @@
FROM python:3.12-slim
# syntax=docker/dockerfile:1.7
#
# Example gateway image: the tenant-facing proxy in front of DSH sandboxes.
# The base image is digest-pinned in versions.env (GATEWAY_PYTHON_IMAGE) and
# scripts/check-versions.sh verifies the ARG default below.
#
# Two stages on that base:
# build installs the fully hash-pinned runtime dependencies into /venv as
# a dedicated non-root UID, so the runtime stage copies a venv that
# contains no root-owned files.
# runtime copies /venv and the app, runs as UID 10000 with HOME=/tmp and no
# writable location of its own: usable with readOnlyRootFilesystem.
ARG GATEWAY_PYTHON_IMAGE=docker.io/python:3.12-slim@sha256:aa943ce411423c7582288f18707ff66eddcbaeb510502d747a72aeb34f30d159
# Read-only-rootfs friendly: no .pyc writes, no home directory writes.
# ---------------------------------------------------------------------------
# Build stage: hash-pinned virtualenv owned by the runtime identity.
# ---------------------------------------------------------------------------
FROM $GATEWAY_PYTHON_IMAGE AS build
ARG APP_UID=10000
ARG APP_GID=10000
# No .pyc writes and no user cache dirs: the runtime image stays read-only.
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
HOME=/tmp
WORKDIR /app
RUN groupadd --gid "$APP_GID" app \
&& useradd --uid "$APP_UID" --gid "$APP_GID" --no-create-home \
--home-dir /tmp --shell /usr/sbin/nologin app \
&& python -m venv /venv \
&& chown -R "$APP_UID:$APP_GID" /venv
WORKDIR /build
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY gateway.py .
# Install as the unprivileged user so every file in /venv is owned by UID 10000.
# --require-hashes makes pip reject any requirement without a matching hash:
# the dependency set in requirements.txt is fully pinned and verified. Use the
# venv's own pip: the system pip would user-install into $HOME instead.
USER $APP_UID:$APP_GID
RUN /venv/bin/pip install --no-cache-dir --require-hashes -r requirements.txt
# ---------------------------------------------------------------------------
# Runtime stage
# ---------------------------------------------------------------------------
FROM $GATEWAY_PYTHON_IMAGE AS runtime
ARG APP_UID=10000
ARG APP_GID=10000
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
HOME=/tmp \
PATH="/venv/bin:$PATH"
RUN groupadd --gid "$APP_GID" app \
&& useradd --uid "$APP_UID" --gid "$APP_GID" --no-create-home \
--home-dir /tmp --shell /usr/sbin/nologin app
WORKDIR /app
# Dependencies come from the build stage: same interpreter, no build tools left
# in the runtime image and nothing owned by root inside /venv.
COPY --from=build --chown=$APP_UID:$APP_GID /venv /venv
COPY --chown=$APP_UID:$APP_GID gateway.py .
EXPOSE 8080
# Run as a fixed non-root UID; HOME=/tmp keeps caches out of the image.
USER 10000

115
examples/dsh-agents-as-a-service/gateway/README.md
View File

@@ -10,11 +10,13 @@ It does four things:
(only its SHA-256 is stored as a claim annotation), reports derived state,
and deletes the claim (cascading the sandbox and PVC).
2. **Browser sessions** — exchanges a user token for an HttpOnly session
cookie so the DSH UI works in a browser at `/u/<user>/`. Automation can
still use `Authorization: Bearer <token>`.
cookie whose value is only an opaque, server-side session id (user and
expiry stay in the gateway's memory) so the DSH UI works in a browser at
`/u/<user>/`. Automation can still use
`Authorization: Bearer <token>`, which never comes from the cookie.
3. **Reverse proxy** — proxies HTTP (including SSE) and WebSocket traffic to
the sandbox's DSH Web server on `:3080`, including DSH's own cookie
handshake.
handshake. Client-supplied forwarding/identity headers are stripped.
4. **Optional SSO** — when `AUTH_PROXY_HEADER` is set, trusts the identity
header an authenticating reverse proxy (for example oauth2-proxy) forwards,
auto-provisions that user, and exposes `POST /recreate` for a clean restart.
@@ -63,10 +65,10 @@ token. A failed handshake returns `503` with `Retry-After: 10`.
| Method | Path | Auth | Purpose |
| --- | --- | --- | --- |
| `GET` | `/healthz` | none | Liveness/readiness. Never touches the cluster. |
| `GET` | `/` | none | Landing page with signup + token login forms. With SSO it auto-provisions the proxy-identified user and redirects to their workspace. |
| `POST` | `/users` | none | Signup. JSON body `{"user": "<dns-1123 label>"}` or form field `user`. JSON returns `201` when Ready, `202` when accepted but still Waking; form returns `303` to `/u/<user>/`. Both set the session cookie. `409` if the user exists, `400` if invalid, `413` if the body is too large. |
| `POST` | `/login` | none | Form (`user`, `token`) returns `303` to `/u/<user>/`; JSON returns `200`. Sets the session cookie. `401` on a bad token, `404` for an unknown user. |
| `POST` | `/logout` | none | Clears the session cookie and `303`s to `/`. |
| `GET` | `/` | none | Landing page with signup + token login forms (plus an invite-code input when `SIGNUP_INVITE_CODE` is set). With SSO it auto-provisions the proxy-identified user and redirects to their workspace. |
| `POST` | `/users` | none | Signup. JSON body `{"user": "<dns-1123 label>"}` or form field `user`; when `SIGNUP_INVITE_CODE` is set, an `invite` form field or `X-Signup-Invite` header must match it (`403` otherwise, checked before the user lookup). JSON returns `201` when Ready, `202` when accepted but still Waking; form returns `303` to `/u/<user>/`. Both set the session cookie. `409` if the user exists, `400` if invalid, `413` if the body is too large, `429` when the auth rate limit is hit. |
| `POST` | `/login` | none | Form (`user`, `token`) returns `303` to `/u/<user>/`; JSON returns `200`. Sets the session cookie. `401` on a bad token **and** for an unknown user (no username enumeration). `429` when the auth rate limit is hit. Both auth routes reject a form post whose `Origin`/`Referer` contradicts `Host` (`403`) when the browser sent one. |
| `POST` | `/logout` | none | Revokes the server-side session, clears the cookie and `303`s to `/`. |
| `POST` | `/recreate` | trusted proxy header | **SSO only.** Deletes the authenticated user's claim (cascading sandbox + PVC, so workspace data is wiped), waits out finalizers, provisions a fresh claim and `303`s to `/u/<user>/`. `401` without a valid identity header; cookie-authenticated calls must be same-origin. |
| `GET` | `/users/{user}` | bearer or cookie | Derived state: `Provisioning`, `Waking`, `Ready`, `Suspending`, `Suspended`. |
| `DELETE` | `/users/{user}` | bearer or cookie | Delete the claim (sandbox + PVC cascade). Cookie-auth mutations require a same-origin `Origin`/`Referer`. |
@@ -78,19 +80,49 @@ token. A failed handshake returns `503` with `Retry-After: 10`.
* `Authorization: Bearer <token>` is checked with a timing-safe SHA-256
comparison against the `aaas.example.com/token-sha256` claim annotation.
* The browser session cookie is
`dsh_gw_session=<user>:<token>` (HttpOnly, SameSite=Lax, Path=/, optional
Secure). A session for user A never authorizes user B.
* The browser session cookie is **opaque**: its value is a random
server-side session id; the user and expiry live in gateway memory
(`GatewayState.sessions`). A cookie can never be edited into another user's
credentials, a session id is not accepted as a bearer token, and a bearer
token never authorizes a different user. The cookie name defaults to
`__Host-dsh_gw_session` when `COOKIE_SECURE=true` (HTTPS) and to
`dsh_gw_session` for plain HTTP; attributes are HttpOnly, SameSite=Lax,
Path=/ and (with `COOKIE_SECURE`) Secure.
* Sessions are revoked on `POST /logout`, on `DELETE /users/{user}`, on
`POST /recreate`, and when they expire (looked up on every request; the
sweeper also drops expired entries). `from_env()` refuses to start without
`COOKIE_SECURE=true` or the explicit `ALLOW_INSECURE_COOKIES=1` opt-out for
local HTTP.
* Cookie-authenticated **mutations** (`POST`, `PUT`, `PATCH`, `DELETE`) and
cookie-authenticated **WebSocket upgrades** require the request `Origin`
(or `Referer` fallback) to match the `Host` header. Missing or mismatched
cross-origin metadata is rejected (`403` HTTP, close code `4401` for WS).
Form signup/login additionally reject a cross-origin `Origin`/`Referer`
when the browser sent one, before the body is read.
* `POST /users` and `POST /login` are rate limited per client:
`AUTH_RATE_LIMIT` attempts per `AUTH_RATE_WINDOW` seconds (default 30/60,
`0` disables). The client key is the rightmost `X-Forwarded-For` entry when
present (the address the last trusted proxy saw, not a spoofable leftmost
value), else the socket peer. Over-limit requests get `429` +
`Retry-After` before the body is read.
* Proxy requests strip client-supplied forwarding/identity headers in both
HTTP and WebSocket paths: `Forwarded`, every `X-Forwarded-*`
(including `X-Forwarded-User`), every `X-Auth-Request-*`, and `X-Real-IP`.
The gateway sets its own `X-Forwarded-For/Host/Proto/Port` values from the
connection. `UPSTREAM_HEADER_ALLOWLIST` optionally narrows the forwarded
client headers further (comma-separated, case-insensitive); unset keeps the
denylist behavior, and the gateway's own values are always added.
* The gateway's bearer token, the browser `Cookie` header and any `?token=`
query parameter are never forwarded upstream. Only the cached `dsh-auth-*`
cookie is injected.
* Upstream `Set-Cookie` headers are dropped. `Location` headers that are
absolute paths are rewritten under `/u/<user>/`, and any `token` query
parameter is stripped.
* Concurrency is capped per user: `MAX_IN_FLIGHT_PER_USER` proxied requests
(default 32) before the gateway answers `503` + `Retry-After` (HTTP) or
closes the WebSocket with `1011`. Messages received from DSH are limited to
`WS_MAX_MESSAGE_BYTES` (default 16 MiB); browser-to-DSH frames are bounded by
uvicorn's own WebSocket size limit.
### SSO (opt-in)
@@ -106,7 +138,8 @@ kubectl -n dsh-demo set env deploy/dsh-gateway AUTH_PROXY_HEADER=X-Forwarded-Use
When set, the gateway treats a request carrying that header as authenticated as
that user, provided the value is a valid DNS-1123 label; the header is never
trusted when the variable is unset. The landing page then auto-provisions the
user (creates their claim idempotently, mints a rotated session token) and
user (creates their claim idempotently, rotates the claim token when a new
session is needed and mints an opaque session cookie) and
redirects to `/u/<user>/`, and `POST /recreate` becomes available for a clean
restart. The base manifests leave it unset on purpose; the
[SSO exposure variant](../manifests/exposure/gateway-api-sso/) documents how to
@@ -133,14 +166,19 @@ oauth2-proxy's egress to the gateway, DNS and public HTTPS.
| --- | --- |
| Unknown user / claim | `404` |
| Missing or invalid credentials | `401` |
| Cross-origin cookie mutation | `403` |
| Unknown user on `/login` | `401` (same as a bad token; no username enumeration) |
| Cross-origin cookie mutation / cross-origin form signup or login | `403` |
| Auth route over `AUTH_RATE_LIMIT` | `429` + `Retry-After`, before the body is read |
| Request body over `MAX_BODY_MB` | `413` |
| Client aborts mid-body, before the response starts | `499` (`CLIENT_CLOSED_REQUEST`). Nothing is forwarded to DSH and nothing can be delivered; it is not an error, so no traceback is logged. Covers the body-reading control routes (`POST /users`, `POST /login`) as well as the proxy. |
| Client aborts mid-stream (SSE, long response) | The response already started, so its upstream status stands (typically `200`); the stream simply ends. No traceback, and the in-flight slot is released as the relay unwinds. |
| Sandbox not provisioned, wake timeout, handshake failure | `503` + `Retry-After: 10` |
| Sandbox not provisioned, wake timeout, handshake failure | `503` + `Retry-After: 10`. The handshake error body is a fixed message: it never contains the launch token, the handshake URL or pod/credential details. |
| User over `MAX_IN_FLIGHT_PER_USER` | `503` + `Retry-After` (HTTP); the existing in-flight requests are untouched |
| Upstream connection/protocol failure | `502` |
| WS auth failure (unknown user, bad token, cross-origin) | accept, then close `4401` |
| WS per-user concurrency cap | accept, then close `1011` |
| WS upstream/handshake failure | accept, then close `1011` |
| WS message from DSH over `WS_MAX_MESSAGE_BYTES` | the upstream/relay closes the connection (websockets' message-too-big path) |
A close code can only be delivered after the WebSocket upgrade is accepted;
Starlette would turn an early close into an HTTP 403, which is why WS auth
@@ -166,17 +204,27 @@ can also be injected into `create_app` for tests.
| `DSH_PORT` | `3080` | Upstream DSH port. |
| `DSH_CONTAINER` | `dsh-web` | Container whose logs carry the launch token. |
| `DSH_LOG_TAIL_LINES` | `2000` | How many log lines to scan for the launch token. |
| `COOKIE_NAME` | `dsh_gw_session` | Browser session cookie name. |
| `COOKIE_SECURE` | `false` | Add `Secure` to the session cookie when served over TLS. |
| `COOKIE_MAX_AGE` | `604800` | Session cookie lifetime in seconds. |
| `COOKIE_NAME` | `__Host-dsh_gw_session` when secure, else `dsh_gw_session` | Browser session cookie name. Only an opaque session id is stored in it. |
| `COOKIE_SECURE` | `false` | Add `Secure` to the session cookie. Required unless `ALLOW_INSECURE_COOKIES=1` is set for local HTTP. |
| `ALLOW_INSECURE_COOKIES` | `0` | Explicit opt-out for plain-HTTP demos. Without it, `Settings.from_env()` refuses to start unless `COOKIE_SECURE=true`. |
| `COOKIE_MAX_AGE` | `604800` | Session lifetime in seconds (server-side expiry and cookie `Max-Age`). |
| `MAX_BODY_MB` | `32` | Maximum buffered request body (and `413` threshold). |
| `HTTP_TIMEOUT` | `300` | Upstream connect/read timeout in seconds. Proxied streams (SSE, long responses) keep the connect/write budget but have **no read timeout**, so an idle event stream is not cut every five minutes. |
| `SWEEP_INTERVAL` | `15` | Seconds between idle sweeps. |
| `SWEEP_INTERVAL` | `15` | Seconds between idle sweeps. Sweeps also prune expired sessions and rate buckets and orphaned per-user state. |
| `SIGNUP_INVITE_CODE` | unset | When set, `POST /users` requires the matching `invite` form field or `X-Signup-Invite` header (timing-safe compare); the landing page then shows an invite input. Unset leaves signup open. |
| `AUTH_RATE_LIMIT` | `30` | Attempts allowed per client per `AUTH_RATE_WINDOW` on `POST /users` and `POST /login`; `0` disables the limit. |
| `AUTH_RATE_WINDOW` | `60` | Rate-limit window in seconds. |
| `UPSTREAM_HEADER_ALLOWLIST` | unset | Comma-separated, case-insensitive allowlist of client headers to forward upstream. Unset keeps the denylist behavior; forwarding/identity headers are stripped in both modes. |
| `MAX_IN_FLIGHT_PER_USER` | `32` | Concurrent proxied requests per user before `503`/WS `1011`. `0` disables the cap. |
| `WS_MAX_MESSAGE_BYTES` | `16777216` | Largest WebSocket message (16 MiB) accepted from DSH. Browser-to-DSH frames are bounded by uvicorn's own limit. |
| `AUTH_PROXY_HEADER` | unset | Opt-in SSO: name of the trusted identity header set by an authenticating reverse proxy (e.g. `X-Forwarded-User`). Unset disables SSO and the header is never trusted. Enable only when the Service is reachable exclusively through that proxy (see [SSO](#sso-opt-in)). |
The shipped `manifests/50-gateway.yaml` sets `IDLE_TIMEOUT=360` and
`WAKE_TIMEOUT=240` for the demo; the e2e script overrides them to 60/120 so the
smoke test does not wait minutes for the sweeper.
smoke test does not wait minutes for the sweeper. It also sets
`COOKIE_SECURE=false` together with the explicit `ALLOW_INSECURE_COOKIES=1`
acknowledgement because the demo is reached over plain HTTP (a `Secure` cookie
would never be sent). HTTPS deployments must set `COOKIE_SECURE=true`.
### Required RBAC
@@ -197,8 +245,10 @@ python -m pip install -r requirements-dev.txt
python -m pytest -q
python -m ruff check .
# Uvicorn factory (loads in-cluster credentials only on the first claim call)
uvicorn gateway:create_app --factory --host 0.0.0.0 --port 8080
# Uvicorn factory (loads in-cluster credentials only on the first claim call).
# from_env() insists on an explicit acknowledgement for plain HTTP; behind TLS
# set COOKIE_SECURE=true instead.
ALLOW_INSECURE_COOKIES=1 uvicorn gateway:create_app --factory --host 0.0.0.0 --port 8080
```
Container image, from the repository root:
@@ -226,18 +276,27 @@ uvicorn's access log so bearer tokens in URLs are not logged.
Log rotation, a very chatty container, or DSH changing its startup message
breaks the handshake. Prefer a first-class token/credential API when DSH
offers one.
* **Single replica, in-memory state.** The handshake cookie cache, idle
activity clocks and in-flight counters live in one process. Running more
than one replica would need shared state (or sticky routing), otherwise each
replica handshakes independently and the sweeper may race another replica.
* **Single replica, in-memory state.** The handshake cookie cache, the opaque
session table, idle activity clocks, in-flight counters and rate-limit
buckets live in one process. Running more than one replica would need shared
state (or sticky routing), otherwise each replica handshakes independently,
the sweeper may race another replica, and a session/rate budget is only seen
by the replica that created it.
* **Sessions die with the process.** An opaque session id is valid only in the
process that minted it and expires after `COOKIE_MAX_AGE`; a restart logs
every browser out (tokens are unaffected).
* **In-memory token cache has no TTL.** It is keyed by pod UID, so pod
restarts invalidate it naturally; otherwise entries survive until the pod or
claim is deleted.
* **No TLS termination.** Run behind an ingress/load balancer; set
`COOKIE_SECURE=true` there.
* **No TLS termination.** Run behind an ingress/load balancer and set
`COOKIE_SECURE=true`. `Settings.from_env()` refuses to start without it
unless `ALLOW_INSECURE_COOKIES=1` explicitly acknowledges plain-HTTP
sessions.
* **Origin-based CSRF protection only.** Cookie-auth mutations require a
same-origin `Origin`/`Referer`; there is no per-form CSRF token, rate
limiting or login throttling.
same-origin `Origin`/`Referer`, and form signup/login reject a contradicting
origin when the browser sends one; there is no per-form CSRF token. Auth
routes have per-client rate limiting and login throttling, but the budget is
in-process (per replica) and keyed by the rightmost `X-Forwarded-For` value.
* **HTTP request bodies are buffered** up to `MAX_BODY_MB`; uploads are not
streamed. Responses (including SSE) are streamed.
* **Upstream cookies are not forwarded.** DSH cannot rotate its own

395
examples/dsh-agents-as-a-service/gateway/gateway.py
View File

@@ -7,9 +7,12 @@ single teaching-friendly module with three jobs:
(only its SHA-256 is stored, as a claim annotation), report derived state,
and delete the claim (which cascades the sandbox and its PVC).
* **Browser sessions** -- exchange a user token for an HttpOnly session cookie
so the DSH UI can be opened in a browser at ``/u/<user>/``. Browser requests
authenticate with that cookie; automation can keep using
``Authorization: Bearer <token>``.
whose value is an opaque, server-side session id (the user and expiry live in
memory) so the DSH UI can be opened in a browser at ``/u/<user>/``. Browser
requests authenticate with that cookie; automation can keep using
``Authorization: Bearer <token>``. Sessions are revoked on logout, user
deletion, recreate and expiry, and a cookie is never accepted as a bearer
token (nor the other way around).
* **Reverse proxy** -- HTTP (including SSE) and WebSocket proxying to the
sandbox's DSH Web server on ``:3080``. DSH itself is cookie-gated: the
gateway scrapes the launch token from the pod logs, performs the ``?token=``
@@ -38,6 +41,7 @@ import contextlib
import hashlib
import html
import json
import math
import os
import re
import secrets
@@ -121,6 +125,19 @@ OWNED_REQUEST_HEADERS = HOP_BY_HOP | {
"x-real-ip",
}
# Default session cookie names. ``__Host-`` is only legal over HTTPS and pins
# the cookie to Path=/ with no Domain attribute; the insecure default keeps the
# example usable over plain HTTP behind a port-forward.
DEFAULT_COOKIE_NAME = "dsh_gw_session"
SECURE_COOKIE_NAME = "__Host-dsh_gw_session"
# Header prefixes a client must never be able to smuggle upstream. The gateway
# derives its own X-Forwarded-* values from the connection; anything the browser
# sends with these names is attacker-controlled and is dropped in both the HTTP
# and WebSocket proxy paths.
FORWARDING_HEADER_PREFIXES = ("x-forwarded-", "x-auth-request-")
FORWARDING_HEADERS = frozenset({"forwarded"})
_DSH_WEB_LINE = re.compile(r"dsh web:\s+(?P<url>https?://\S+)")
@@ -249,6 +266,33 @@ def same_origin(headers) -> bool:
return False
def strip_forwarding_header(name: str) -> bool:
"""True for client-supplied identity/forwarding headers we never forward.
``X-Forwarded-User`` and ``X-Auth-Request-*`` carry identities an
authenticating proxy or the gateway itself owns; ``Forwarded`` /
``X-Forwarded-*`` describe the hop chain. Forwarding any of them lets a
browser spoof its origin or a trusted SSO user upstream.
"""
lowered = name.lower()
return lowered in FORWARDING_HEADERS or lowered.startswith(FORWARDING_HEADER_PREFIXES)
def client_identity(headers, client) -> str:
"""The rate-limit bucket for a request.
The rightmost ``X-Forwarded-For`` entry is the address the last trusted
proxy saw (each proxy appends), so it cannot be spoofed by prepending
arbitrary values. Without the header fall back to the socket peer.
"""
forwarded = (headers.get("x-forwarded-for") or "").strip()
if forwarded:
parts = [part.strip() for part in forwarded.split(",") if part.strip()]
if parts:
return parts[-1]
return getattr(client, "host", None) or "unknown"
def rewrite_location(location: str, user: str) -> str:
"""Prefix absolute-path redirects so the browser stays under ``/u/<user>/``."""
if not location.startswith("/") or location.startswith("//"):
@@ -306,6 +350,14 @@ def _env_float(name: str, default: float) -> float:
return default
def _env_allowlist(name: str) -> frozenset[str] | None:
"""Parse a comma-separated header allowlist; unset means "denylist mode"."""
raw = os.environ.get(name)
if raw is None:
return None
return frozenset(part.strip().lower() for part in raw.split(",") if part.strip())
# --- Settings --------------------------------------------------------------
@@ -325,7 +377,7 @@ class Settings:
dsh_port: int = 3080
dsh_container: str = "dsh-web"
dsh_log_tail_lines: int = 2000
cookie_name: str = "dsh_gw_session"
cookie_name: str = DEFAULT_COOKIE_NAME
cookie_secure: bool = False
cookie_max_age: int = 7 * 24 * 60 * 60
max_body_bytes: int = 32 * 1024 * 1024
@@ -339,9 +391,36 @@ class Settings:
# valid user in this header are treated as authenticated (SSO); unset (None)
# disables SSO entirely and the header is never trusted.
auth_proxy_header: str | None = None
# Abuse controls. AUTH_RATE_LIMIT applies to the unauthenticated auth routes
# (POST /users and POST /login) per client and window; 0 disables it.
auth_rate_limit: int = 30
auth_rate_window: int = 60
# When set, signup requires this invite code (form field ``invite`` or the
# ``X-Signup-Invite`` header). None keeps signup open, as in the demo.
signup_invite_code: str | None = None
# Optional allowlist of client request headers to forward upstream. None
# keeps the default denylist behavior; a set (possibly empty) forwards only
# the named headers (plus the gateway's own X-Forwarded-* values).
upstream_header_allowlist: frozenset[str] | None = None
# Concurrent proxied requests per user before 503/1011 is returned.
max_in_flight_per_user: int = 32
# Largest WebSocket message accepted from DSH, in bytes. Browser-to-DSH
# frames are bounded by the ASGI server's own WebSocket limit.
ws_max_message_bytes: int = 16 * 1024 * 1024
@classmethod
def from_env(cls) -> Settings:
cookie_secure = _env_bool("COOKIE_SECURE", False)
allow_insecure = _env_bool("ALLOW_INSECURE_COOKIES", False)
if not cookie_secure and not allow_insecure:
raise RuntimeError(
"refusing to start with an insecure session cookie: set "
"COOKIE_SECURE=true behind TLS, or ALLOW_INSECURE_COOKIES=1 "
"for a local plain-HTTP demo"
)
cookie_name = os.environ.get("COOKIE_NAME") or (
SECURE_COOKIE_NAME if cookie_secure else DEFAULT_COOKIE_NAME
)
return cls(
namespace=os.environ.get("NAMESPACE", "dsh-demo"),
pool=os.environ.get("POOL", "dsh-pool"),
@@ -350,13 +429,19 @@ class Settings:
dsh_port=_env_int("DSH_PORT", 3080),
dsh_container=os.environ.get("DSH_CONTAINER", "dsh-web"),
dsh_log_tail_lines=_env_int("DSH_LOG_TAIL_LINES", 2000),
cookie_name=os.environ.get("COOKIE_NAME", "dsh_gw_session"),
cookie_secure=_env_bool("COOKIE_SECURE", False),
cookie_name=cookie_name,
cookie_secure=cookie_secure,
cookie_max_age=_env_int("COOKIE_MAX_AGE", 7 * 24 * 60 * 60),
max_body_bytes=_env_int("MAX_BODY_MB", 32) * 1024 * 1024,
http_timeout=_env_float("HTTP_TIMEOUT", 300.0),
sweep_interval=_env_float("SWEEP_INTERVAL", 15.0),
auth_proxy_header=os.environ.get("AUTH_PROXY_HEADER") or None,
auth_rate_limit=_env_int("AUTH_RATE_LIMIT", 30),
auth_rate_window=_env_int("AUTH_RATE_WINDOW", 60),
signup_invite_code=os.environ.get("SIGNUP_INVITE_CODE") or None,
upstream_header_allowlist=_env_allowlist("UPSTREAM_HEADER_ALLOWLIST"),
max_in_flight_per_user=_env_int("MAX_IN_FLIGHT_PER_USER", 32),
ws_max_message_bytes=_env_int("WS_MAX_MESSAGE_BYTES", 16 * 1024 * 1024),
)
@@ -546,6 +631,19 @@ class Kubernetes:
# --- Gateway ---------------------------------------------------------------
@dataclass(frozen=True)
class Session:
"""A browser session stored server-side, keyed by an opaque random id.
The cookie only carries the id; the user and the expiry stay in memory, so
a cookie can never be edited into another user's credentials and a stolen
token is not recoverable from the browser.
"""
user: str
expires_at: float
@dataclass
class GatewayState:
"""In-memory, per-process state (single replica by design)."""
@@ -555,6 +653,10 @@ class GatewayState:
maintenance: dict[str, bool] = field(default_factory=dict)
# (user, pod UID, browser Host) -> "dsh-auth-a=...; dsh-auth-b=..."
handshake: dict[tuple[str, str, str], str] = field(default_factory=dict)
# Opaque session id -> server-side session (user + expiry).
sessions: dict[str, Session] = field(default_factory=dict)
# Auth rate limiting: client -> (window start, attempts in window).
auth_rate: dict[str, tuple[float, int]] = field(default_factory=dict)
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
@@ -576,17 +678,40 @@ class Gateway:
# -- credentials -------------------------------------------------------
def session_for(self, sid: str | None) -> Session | None:
"""Look up a live session, dropping it when it has expired."""
if not sid:
return None
session = self.state.sessions.get(sid)
if session is None:
return None
if session.expires_at <= time.time():
self.state.sessions.pop(sid, None)
return None
return session
def revoke_session(self, sid: str | None) -> None:
if sid:
self.state.sessions.pop(sid, None)
def revoke_sessions(self, user: str) -> None:
for sid in [key for key, session in self.state.sessions.items() if session.user == user]:
self.state.sessions.pop(sid, None)
def credentials(self, headers, cookies) -> tuple[str | None, str | None]:
"""Return ``(token, cookie_user)``; bearer wins over the session cookie."""
"""Return ``(token, session_user)``; bearer wins over the session cookie.
A bearer token is never resolved from a cookie and a session id is never
accepted as a bearer token. An unknown or expired session is treated as
anonymous (and the expired entry is dropped).
"""
auth = headers.get("authorization", "")
if auth.lower().startswith("bearer "):
return auth[7:].strip(), None
raw = cookies.get(self.settings.cookie_name)
if raw:
user, sep, token = raw.partition(":")
if sep and user and token:
return token, user
return None, None
session = self.session_for(cookies.get(self.settings.cookie_name))
if session is None:
return None, None
return None, session.user
def remote_user(self, headers) -> str | None:
"""The user asserted by a trusted reverse proxy, when SSO is enabled.
@@ -605,31 +730,37 @@ class Gateway:
self,
claim: dict,
token: str | None,
cookie_user: str | None,
session_user: str | None,
user: str,
remote: str | None = None,
) -> bool:
if remote == user:
return True # identity asserted by the authenticating proxy
if cookie_user is not None and cookie_user != user:
return False # a session for another user never authorizes this one
if session_user is not None:
return session_user == user # server-side session bound to this user
return token_matches(claim, token)
async def _sso_session(self, remote: str, headers, cookies, response: Response) -> dict | None:
"""Provision ``remote`` idempotently and mint/refresh a session token.
"""Provision ``remote`` idempotently and mint/refresh a session.
The authenticating proxy has already verified ``remote``, so it is
trusted. A brand-new user is provisioned (claim created, token minted);
an existing user whose session cookie is missing or no longer matches
gets a rotated token so a fresh cookie can be issued. A still-valid
session is reused untouched. Provisioning is non-blocking: the sandbox
is awaited by the proxy path (``_resolve_sandbox``), so the landing page
can report "Provisioning" instead of stalling for ``WAKE_TIMEOUT``.
an existing user whose session cookie is missing, foreign or no longer
matches gets a rotated token so a fresh cookie can be issued. A still-valid
session for the same user is reused untouched. Provisioning is
non-blocking: the sandbox is awaited by the proxy path
(``_resolve_sandbox``), so the landing page can report "Provisioning"
instead of stalling for ``WAKE_TIMEOUT``.
"""
claim = await self.k8s.get_claim(remote)
token, cookie_user = self.credentials(headers, cookies)
if claim is not None and cookie_user == remote and token_matches(claim, token):
return claim
sid = cookies.get(self.settings.cookie_name)
session = self.session_for(sid)
if session is not None:
if session.user != remote:
# A session for another identity never authorizes this user.
self.revoke_session(sid)
elif claim is not None:
return claim
token = secrets.token_urlsafe(32)
token_hash = hashlib.sha256(token.encode()).hexdigest()
is_new = claim is None
@@ -645,7 +776,7 @@ class Gateway:
else:
await self.k8s.patch_token(remote, token_hash)
await self._touch(remote)
self.set_session(response, remote, token)
self.set_session(response, remote)
return await self.k8s.get_claim(remote) if is_new else claim
@staticmethod
@@ -656,16 +787,73 @@ class Gateway:
and not same_origin(headers)
)
def set_session(self, response: Response, user: str, token: str) -> None:
def set_session(self, response: Response, user: str) -> str:
"""Mint an opaque server-side session and set the browser cookie."""
sid = secrets.token_urlsafe(32)
self.state.sessions[sid] = Session(
user=user,
expires_at=time.time() + self.settings.cookie_max_age,
)
response.set_cookie(
self.settings.cookie_name,
f"{user}:{token}",
sid,
max_age=self.settings.cookie_max_age,
path="/",
httponly=True,
samesite="lax",
secure=self.settings.cookie_secure,
)
return sid
# -- abuse controls ----------------------------------------------------
def _rate_limit(self, request: Request) -> Response | None:
"""Return a 429 response when this client is over the auth budget."""
limit = self.settings.auth_rate_limit
if limit <= 0:
return None
window = max(1, self.settings.auth_rate_window)
key = client_identity(request.headers, request.client)
now = time.time()
bucket = self.state.auth_rate.get(key)
if bucket is None or now - bucket[0] >= window:
self.state.auth_rate[key] = (now, 1)
return None
start, count = bucket
if count >= limit:
retry_after = max(1, math.ceil(start + window - now))
return self._error(
429,
"too many authentication attempts, retry shortly",
{"Retry-After": str(retry_after)},
)
self.state.auth_rate[key] = (start, count + 1)
return None
def _invite_allowed(self, request: Request, payload: dict) -> bool:
"""Check the signup invite code, when one is configured."""
expected = self.settings.signup_invite_code
if expected is None:
return True
provided = payload.get("invite")
if not isinstance(provided, str) or not provided:
provided = request.headers.get("x-signup-invite", "")
if not isinstance(provided, str) or not provided:
return False
return secrets.compare_digest(provided.encode("utf-8"), expected.encode("utf-8"))
def _form_cross_origin(self, request: Request) -> bool:
"""Reject a form signup/login whose Origin/Referer contradicts Host.
Only checked when the browser actually sent one: non-browser clients
(automation, tests) commonly omit both headers and must keep working.
"""
if not self._wants_form(request):
return False
headers = request.headers
if not (headers.get("origin") or headers.get("referer")):
return False
return not same_origin(headers)
# -- activity / maintenance -------------------------------------------
@@ -674,12 +862,21 @@ class Gateway:
self.state.last_activity[user] = time.time()
async def _acquire(self, user: str) -> bool:
"""Reserve an in-flight slot, waiting out a concurrent suspend."""
"""Reserve an in-flight slot, waiting out a concurrent suspend.
Returns False when the user already has ``MAX_IN_FLIGHT_PER_USER``
requests in flight (an abuse guard, not a transient error) or when a
suspend does not finish within ``WAKE_TIMEOUT``.
"""
deadline = time.monotonic() + self.settings.wake_timeout
cap = self.settings.max_in_flight_per_user
while True:
async with self.state.lock:
if not self.state.maintenance.get(user):
self.state.in_flight[user] = self.state.in_flight.get(user, 0) + 1
current = self.state.in_flight.get(user, 0)
if cap > 0 and current >= cap:
return False
self.state.in_flight[user] = current + 1
self.state.last_activity[user] = time.time()
return True
if time.monotonic() > deadline:
@@ -759,14 +956,19 @@ class Gateway:
host: str,
force: bool = False,
) -> tuple[str, tuple[str, str, str], bool]:
"""Return ``(cookie, cache_key, was_cached)``, handshaking on a miss."""
"""Return ``(cookie, cache_key, was_cached)``, handshaking on a miss.
Every failure raises ``HandshakeError`` with a fixed, credential-free
message: the exception reaches the browser in the 503 body, so it must
never carry the launch token, the handshake URL or pod details.
"""
pod_name = self._pod_name(sandbox)
try:
pod = await self.k8s.get_pod(pod_name)
except ApiException as exc:
raise HandshakeError(f"could not read pod {pod_name}: {exc}") from exc
raise HandshakeError("could not read the DSH handshake pod") from exc
if pod is None:
raise HandshakeError(f"pod {pod_name} not found")
raise HandshakeError("the DSH handshake pod is not available")
uid = (pod.get("metadata") or {}).get("uid") or pod_name
key = (user, uid, host)
if not force:
@@ -780,7 +982,7 @@ class Gateway:
self.settings.dsh_log_tail_lines,
)
except ApiException as exc:
raise HandshakeError(f"could not read logs for {pod_name}: {exc}") from exc
raise HandshakeError("could not read the pod logs for the handshake") from exc
launch_token = parse_dsh_launch_token(logs)
if not launch_token:
raise HandshakeError("DSH launch token not found in pod logs")
@@ -798,12 +1000,12 @@ class Gateway:
timeout=self.settings.http_timeout,
)
except httpx.HTTPError as exc:
raise HandshakeError(f"DSH handshake failed: {exc}") from exc
raise HandshakeError("the DSH handshake request failed") from exc
if response.status_code >= 400:
raise HandshakeError(f"DSH handshake returned {response.status_code}")
raise HandshakeError("the DSH handshake returned an error status")
cookie = extract_dsh_cookies(response.headers.get_list("set-cookie"))
if not cookie:
raise HandshakeError("DSH handshake set no dsh-auth cookie")
raise HandshakeError("the DSH handshake set no auth cookie")
self.state.handshake[key] = cookie
return cookie, key, False
@@ -879,9 +1081,9 @@ class Gateway:
if "set-cookie" in response.headers:
page.headers["set-cookie"] = response.headers["set-cookie"]
return page
token, user = self.credentials(request.headers, request.cookies)
_token, user = self.credentials(request.headers, request.cookies)
session = ""
if user and token:
if user:
safe_user = html.escape(user)
session = (
f'<p class="session">Signed in as <strong>{safe_user}</strong>. '
@@ -889,6 +1091,11 @@ class Gateway:
'<form method="post" action="/logout">'
'<button type="submit">Sign out</button></form>'
)
invite_input = (
'<input name="invite" type="password" placeholder="invite code" required>'
if self.settings.signup_invite_code is not None
else ""
)
body = f"""<!doctype html>
<html lang="en">
<head>
@@ -911,6 +1118,7 @@ class Gateway:
<h2>Create a sandbox</h2>
<form method="post" action="/users">
<input name="user" placeholder="username (dns-1123 label)" required>
{invite_input}
<button type="submit">Create</button>
</form>
</section>
@@ -970,11 +1178,18 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
# -- users -------------------------------------------------------------
async def signup(self, request: Request) -> Response:
limited = self._rate_limit(request)
if limited is not None:
return limited
if self._form_cross_origin(request):
return self._error(403, "cross-origin request rejected")
wants_form = self._wants_form(request)
try:
payload = await self._read_payload(request)
except BodyTooLarge:
return self._error(413, "request body too large")
if not self._invite_allowed(request, payload):
return self._error(403, "invalid or missing invite code")
user = payload.get("user", "")
if not valid_user(user):
return self._error(400, "body must be {'user': '<dns-1123 label>'}")
@@ -992,16 +1207,21 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
sandbox_name = (sandbox or {}).get("metadata", {}).get("name")
if wants_form:
response = RedirectResponse(f"/u/{user}/", status_code=303)
self.set_session(response, user, token)
self.set_session(response, user)
return response
response = JSONResponse(
{"user": user, "token": token, "state": state, "sandbox": sandbox_name},
status_code=201 if state == "Ready" else 202,
)
self.set_session(response, user, token)
self.set_session(response, user)
return response
async def login(self, request: Request) -> Response:
limited = self._rate_limit(request)
if limited is not None:
return limited
if self._form_cross_origin(request):
return self._error(403, "cross-origin request rejected")
wants_form = self._wants_form(request)
try:
payload = await self._read_payload(request)
@@ -1012,21 +1232,29 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
if not isinstance(user, str) or not isinstance(token, str):
return self._error(400, "user and token are required")
claim = await self.k8s.get_claim(user)
if claim is None:
return self._error(404, "not found")
if not token_matches(claim, token):
# An unknown user is indistinguishable from a bad token to the client:
# never reveal which usernames exist.
if claim is None or not token_matches(claim, token):
return self._error(401, "unauthorized")
await self._touch(user)
if wants_form:
response = RedirectResponse(f"/u/{user}/", status_code=303)
else:
response = JSONResponse({"user": user, "state": "ok"})
self.set_session(response, user, token)
self.set_session(response, user)
return response
async def logout(self, request: Request) -> Response:
self.revoke_session(request.cookies.get(self.settings.cookie_name))
response = RedirectResponse("/", status_code=303)
response.delete_cookie(self.settings.cookie_name, path="/")
# Match the attributes the cookie was set with: a __Host- cookie
# deletion that lacks Secure is rejected by browsers, leaving a stale
# (though already revoked) cookie behind.
response.delete_cookie(
self.settings.cookie_name,
path="/",
secure=self.settings.cookie_secure,
)
return response
async def get_user(self, request: Request) -> Response:
@@ -1061,6 +1289,7 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
if self._mutation_from_cookie(request.headers, "DELETE", cookie_user):
return self._error(403, "cross-origin request rejected")
await self.k8s.delete_claim(user)
self.revoke_sessions(user)
self._clear_handshake(user)
async with self.state.lock:
self.state.last_activity.pop(user, None)
@@ -1084,6 +1313,7 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
if self._mutation_from_cookie(request.headers, "POST", cookie_user):
return self._error(403, "cross-origin request rejected")
await self.k8s.delete_claim(remote)
self.revoke_sessions(remote)
self._clear_handshake(remote)
async with self.state.lock:
self.state.last_activity.pop(remote, None)
@@ -1111,7 +1341,7 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
return RedirectResponse("/", status_code=303)
return self._error(404, "unknown user")
token, cookie_user = self.credentials(request.headers, request.cookies)
if remote == user and not (cookie_user == user and token_matches(claim, token)):
if remote == user and cookie_user != user:
# SSO user without a valid session cookie: bootstrap, then continue.
return RedirectResponse("/", status_code=303)
if not self._authorized(claim, token, cookie_user, user, remote):
@@ -1171,11 +1401,17 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
if query:
url = f"{url}?{query}"
headers = {
key: value
for key, value in request.headers.items()
if key.lower() not in OWNED_REQUEST_HEADERS and key.lower() != "content-length"
}
headers = {}
allowlist = self.settings.upstream_header_allowlist
for key, value in request.headers.items():
lowered = key.lower()
if lowered in OWNED_REQUEST_HEADERS or lowered == "content-length":
continue
if strip_forwarding_header(lowered):
continue # client-supplied identity/hop-chain headers are spoofable
if allowlist is not None and lowered not in allowlist:
continue
headers[key] = value
host = request.headers.get("host", "")
headers["host"] = host
headers["cookie"] = cookie # only the cached DSH auth cookie
@@ -1309,13 +1545,19 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
query = filtered_query(websocket.url.query)
if query:
url = f"{url}?{query}"
headers = {
key: value
for key, value in websocket.headers.items()
if key.lower() not in OWNED_REQUEST_HEADERS
and key.lower() != "origin"
and not key.lower().startswith("sec-websocket-")
}
headers = {}
allowlist = self.settings.upstream_header_allowlist
for key, value in websocket.headers.items():
lowered = key.lower()
if lowered in OWNED_REQUEST_HEADERS or lowered == "origin":
continue
if lowered.startswith("sec-websocket-"):
continue
if strip_forwarding_header(lowered):
continue # client-supplied identity/hop-chain headers
if allowlist is not None and lowered not in allowlist:
continue
headers[key] = value
headers["cookie"] = cookie
headers["x-forwarded-host"] = host
headers["x-forwarded-proto"] = websocket.url.scheme
@@ -1331,7 +1573,7 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
additional_headers=headers,
subprotocols=subprotocols or None,
open_timeout=self.settings.wake_timeout,
max_size=None,
max_size=self.settings.ws_max_message_bytes,
proxy=None,
)
upstream = await upstream_cm.__aenter__()
@@ -1391,8 +1633,41 @@ against the JSON API. The browser session is an HttpOnly cookie.</p>
# -- idle sweeper ------------------------------------------------------
def _prune_state(self, claims: list[dict]) -> None:
"""Drop state that can no longer be reached.
Sessions and activity clocks for a user whose claim is gone, expired
sessions and expired rate-limit buckets would otherwise accumulate for
the lifetime of the process. In-flight counters are only pruned at
zero, and a user with work in flight is always preserved.
"""
now = time.time()
present = set()
for claim in claims:
name = (claim.get("metadata") or {}).get("name") or ""
if name.startswith("dsh-"):
present.add(name[len("dsh-") :])
for user in [user for user in self.state.last_activity if user not in present]:
self.state.last_activity.pop(user, None)
for user, count in list(self.state.in_flight.items()):
if user not in present and count == 0:
self.state.in_flight.pop(user, None)
for user, flag in list(self.state.maintenance.items()):
if user not in present and not flag:
self.state.maintenance.pop(user, None)
for key in [key for key in self.state.handshake if key[0] not in present]:
self.state.handshake.pop(key, None)
for sid, session in list(self.state.sessions.items()):
if session.user not in present or session.expires_at <= now:
self.state.sessions.pop(sid, None)
window = max(1, self.settings.auth_rate_window)
for key, (start, _count) in list(self.state.auth_rate.items()):
if now - start >= window:
self.state.auth_rate.pop(key, None)
async def sweep_once(self) -> None:
claims = await self.k8s.list_claims()
self._prune_state(claims)
for claim in claims:
# Only suspend claims this gateway created. Everything else in
# the namespace (resource-layer claims from the README walkthrough,

1745
examples/dsh-agents-as-a-service/gateway/requirements-dev.txt
View File

File diff suppressed because it is too large Load Diff

1587
examples/dsh-agents-as-a-service/gateway/requirements.txt
View File

File diff suppressed because it is too large Load Diff

680
examples/dsh-agents-as-a-service/gateway/test_gateway.py
View File

@@ -18,10 +18,12 @@ from types import SimpleNamespace
import httpx
import pytest
from kubernetes.client.exceptions import ApiException
from starlette.responses import Response
from starlette.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
from gateway import (
Session,
Settings,
UserExists,
claim_name,
@@ -219,6 +221,39 @@ def add_user(k8s, user="alice", token=TOKEN, ready=True, pod_name=None, suffix="
return sandbox_name, pod_name
# --- browser sessions ------------------------------------------------------
#
# A browser session is an opaque server-side id. Tests must never forge a
# cookie (or a bearer token); they mint a real session through the gateway so
# the production lookup path is exercised.
def mint_session(env, user="alice") -> tuple[str, str]:
"""Mint a server-side session for ``user``; returns ``(cookie_name, sid)``."""
response = Response()
sid = env.gateway.set_session(response, user)
return env.gateway.settings.cookie_name, sid
def session_dict(env, user="alice") -> dict:
name, sid = mint_session(env, user)
return {name: sid}
def session_cookie(env, user="alice") -> str:
name, sid = mint_session(env, user)
return f"{name}={sid}"
def response_session_user(env, response) -> str:
"""The user of the server-side session a response just set."""
cookie = response.headers["set-cookie"]
value = cookie.split(";", 1)[0].split("=", 1)[1]
session = env.gateway.state.sessions.get(value)
assert session is not None, "response did not set a server-side session"
return session.user
@pytest.fixture
def env():
settings = Settings(
@@ -463,6 +498,12 @@ def test_settings_from_env(monkeypatch):
monkeypatch.setenv("MAX_BODY_MB", "2")
monkeypatch.setenv("HTTP_TIMEOUT", "12.5")
monkeypatch.setenv("AUTH_PROXY_HEADER", "X-Forwarded-User")
monkeypatch.setenv("AUTH_RATE_LIMIT", "5")
monkeypatch.setenv("AUTH_RATE_WINDOW", "30")
monkeypatch.setenv("SIGNUP_INVITE_CODE", "invite-x")
monkeypatch.setenv("UPSTREAM_HEADER_ALLOWLIST", "X-Custom, Accept")
monkeypatch.setenv("MAX_IN_FLIGHT_PER_USER", "4")
monkeypatch.setenv("WS_MAX_MESSAGE_BYTES", "1024")
settings = Settings.from_env()
assert settings.namespace == "ns-x"
assert settings.pool == "pool-x"
@@ -476,13 +517,56 @@ def test_settings_from_env(monkeypatch):
assert settings.max_body_bytes == 2 * 1024 * 1024
assert settings.http_timeout == 12.5
assert settings.auth_proxy_header == "X-Forwarded-User"
assert settings.auth_rate_limit == 5
assert settings.auth_rate_window == 30
assert settings.signup_invite_code == "invite-x"
assert settings.upstream_header_allowlist == frozenset({"x-custom", "accept"})
assert settings.max_in_flight_per_user == 4
assert settings.ws_max_message_bytes == 1024
def test_settings_auth_proxy_header_defaults_to_none(monkeypatch):
monkeypatch.delenv("AUTH_PROXY_HEADER", raising=False)
monkeypatch.setenv("ALLOW_INSECURE_COOKIES", "1")
assert Settings.from_env().auth_proxy_header is None
def test_settings_from_env_rejects_insecure_cookies_by_default(monkeypatch):
monkeypatch.delenv("COOKIE_SECURE", raising=False)
monkeypatch.delenv("ALLOW_INSECURE_COOKIES", raising=False)
with pytest.raises(RuntimeError, match="insecure session cookie"):
Settings.from_env()
def test_settings_from_env_allows_insecure_cookies_with_the_explicit_flag(monkeypatch):
monkeypatch.delenv("COOKIE_SECURE", raising=False)
monkeypatch.setenv("ALLOW_INSECURE_COOKIES", "1")
monkeypatch.delenv("COOKIE_NAME", raising=False)
settings = Settings.from_env()
assert settings.cookie_secure is False
assert settings.cookie_name == "dsh_gw_session"
def test_settings_default_cookie_name_is_host_prefixed_when_secure(monkeypatch):
monkeypatch.setenv("COOKIE_SECURE", "true")
monkeypatch.delenv("COOKIE_NAME", raising=False)
assert Settings.from_env().cookie_name == "__Host-dsh_gw_session"
def test_settings_explicit_cookie_name_wins(monkeypatch):
monkeypatch.setenv("COOKIE_SECURE", "true")
monkeypatch.setenv("COOKIE_NAME", "sid")
assert Settings.from_env().cookie_name == "sid"
def test_settings_upstream_header_allowlist_unset_means_denylist_mode(monkeypatch):
monkeypatch.setenv("ALLOW_INSECURE_COOKIES", "1")
monkeypatch.delenv("UPSTREAM_HEADER_ALLOWLIST", raising=False)
assert Settings.from_env().upstream_header_allowlist is None
monkeypatch.setenv("UPSTREAM_HEADER_ALLOWLIST", "")
assert Settings.from_env().upstream_header_allowlist == frozenset()
# --- Kubernetes adapter ----------------------------------------------------
@@ -635,11 +719,22 @@ async def test_landing_page_has_signup_and_login_forms(env):
assert "text/html" in response.headers["content-type"]
assert 'action="/users"' in response.text
assert 'action="/login"' in response.text
# No invite code configured => no invite input on the landing page.
assert 'name="invite"' not in response.text
@pytest.mark.asyncio
async def test_landing_page_shows_invite_input_only_when_configured(env):
configure(env, signup_invite_code="let-me-in")
async with asgi_client(env) as client:
response = await client.get("/")
assert response.status_code == 200
assert 'name="invite"' in response.text
@pytest.mark.asyncio
async def test_landing_page_shows_session_when_cookie_present(env):
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/")
assert "Signed in as" in response.text
assert "/u/alice/" in response.text
@@ -708,11 +803,14 @@ async def test_signup_json_stores_only_hash_and_returns_token(env, monkeypatch):
assert pool == "test-pool"
assert token_hash == hashlib.sha256(body["token"].encode()).hexdigest()
assert token_hash != body["token"]
# The JSON API also gets a browser session cookie (HttpOnly, SameSite=Lax).
# The JSON API also gets a browser session cookie (HttpOnly, SameSite=Lax)
# carrying only an opaque server-side session id, never the user/token.
cookie = response.headers["set-cookie"]
assert "dsh_gw_session=alice:" in cookie
assert "dsh_gw_session=" in cookie
assert f"dsh_gw_session={TOKEN}" not in cookie
assert "HttpOnly" in cookie
assert "SameSite=lax" in cookie
assert response_session_user(env, response) == "alice"
@pytest.mark.asyncio
@@ -741,13 +839,7 @@ async def test_signup_form_redirects_and_sets_session_cookie(env, monkeypatch):
)
assert response.status_code == 303
assert response.headers["location"] == "/u/bob/"
cookie = response.headers["set-cookie"]
assert re_cookie_user(cookie) == "bob"
def re_cookie_user(set_cookie: str) -> str:
value = set_cookie.split(";", 1)[0].split("=", 1)[1]
return value.split(":", 1)[0]
assert response_session_user(env, response) == "bob"
@pytest.mark.asyncio
@@ -774,7 +866,29 @@ async def test_login_form_sets_cookie_and_redirects(env):
)
assert response.status_code == 303
assert response.headers["location"] == "/u/alice/"
assert re_cookie_user(response.headers["set-cookie"]) == "alice"
assert response_session_user(env, response) == "alice"
@pytest.mark.asyncio
async def test_secure_session_cookie_uses_the_host_prefix_and_flags(env):
configure(env, cookie_secure=True, cookie_name="__Host-dsh_gw_session")
add_user(env.k8s)
async with asgi_client(env) as client:
response = await client.post(
"/login",
data={"user": "alice", "token": TOKEN},
headers={"content-type": "application/x-www-form-urlencoded"},
)
cookie = response.headers["set-cookie"]
assert cookie.startswith("__Host-dsh_gw_session=")
assert "Secure" in cookie
assert "HttpOnly" in cookie
assert "Path=/" in cookie
assert "SameSite=lax" in cookie
assert "Domain=" not in cookie
# The value is an opaque session id, not <user>:<token>.
assert "alice" not in cookie.split(";", 1)[0]
assert TOKEN not in cookie
@pytest.mark.asyncio
@@ -786,17 +900,197 @@ async def test_login_json_and_failures(env):
missing = await client.post("/login", json={"user": "ghost", "token": TOKEN})
assert ok.status_code == 200
assert bad.status_code == 401
assert missing.status_code == 404
# An unknown user is a 401 too: login must not reveal which users exist.
assert missing.status_code == 401
@pytest.mark.asyncio
async def test_logout_clears_cookie(env):
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async def test_logout_clears_cookie_and_revokes_the_session(env):
add_user(env.k8s)
name, sid = mint_session(env)
assert env.gateway.session_for(sid) is not None
async with asgi_client(env, cookies={name: sid}) as client:
response = await client.post("/logout")
assert response.status_code == 303
assert response.headers["location"] == "/"
assert "dsh_gw_session=" in response.headers["set-cookie"]
assert "Max-Age=0" in response.headers["set-cookie"]
# The old sid is dead server-side, not just deleted from the browser.
assert env.gateway.session_for(sid) is None
async with asgi_client(env, cookies={name: sid}) as client:
replay = await client.get("/users/alice")
assert replay.status_code == 401
@pytest.mark.asyncio
async def test_logout_deletion_cookie_is_secure_for_the_host_prefix(env):
"""A __Host- cookie can only be cleared by a Secure deletion Set-Cookie."""
configure(env, cookie_secure=True, cookie_name="__Host-dsh_gw_session")
add_user(env.k8s)
name, sid = mint_session(env)
assert name == "__Host-dsh_gw_session"
async with asgi_client(env, cookies={name: sid}) as client:
response = await client.post("/logout")
assert response.status_code == 303
cookie = response.headers["set-cookie"]
assert cookie.startswith("__Host-dsh_gw_session=")
assert "Max-Age=0" in cookie
assert "Path=/" in cookie
assert "Secure" in cookie
assert "Domain=" not in cookie
# --- signup invite / auth abuse controls -----------------------------------
def configure(env, **overrides):
"""Return a copy of the fixture settings with overrides applied."""
env.settings = Settings(**{**env.settings.__dict__, **overrides})
env.gateway.settings = env.settings
return env
@pytest.mark.asyncio
async def test_signup_requires_invite_when_configured(env, monkeypatch):
configure(env, signup_invite_code="invite-code")
async def not_ready(user, timeout):
return None
monkeypatch.setattr(env.gateway, "wait_ready", not_ready)
async with asgi_client(env) as client:
missing = await client.post("/users", json={"user": "alice"})
wrong = await client.post("/users", json={"user": "alice", "invite": "nope"})
good = await client.post("/users", json={"user": "alice", "invite": "invite-code"})
assert missing.status_code == 403
assert wrong.status_code == 403
assert good.status_code == 202
assert [created[0] for created in env.k8s.created] == ["alice"]
@pytest.mark.asyncio
async def test_invite_is_checked_before_the_user_lookup(env):
configure(env, signup_invite_code="invite-code")
add_user(env.k8s)
async with asgi_client(env) as client:
response = await client.post("/users", json={"user": "alice"})
# 403, not 409: an unauthenticated caller cannot probe for existing users.
assert response.status_code == 403
@pytest.mark.asyncio
async def test_invite_accepts_the_header_and_the_form_field(env, monkeypatch):
configure(env, signup_invite_code="invite-code")
async def not_ready(user, timeout):
return None
monkeypatch.setattr(env.gateway, "wait_ready", not_ready)
async with asgi_client(env) as client:
via_header = await client.post(
"/users",
json={"user": "alice"},
headers={"X-Signup-Invite": "invite-code"},
)
via_form = await client.post(
"/users",
data={"user": "bob", "invite": "invite-code"},
headers={"content-type": "application/x-www-form-urlencoded"},
)
assert via_header.status_code == 202
assert via_form.status_code == 303
@pytest.mark.asyncio
async def test_form_auth_rejects_cross_origin_requests(env):
add_user(env.k8s)
form = {"content-type": "application/x-www-form-urlencoded"}
async with asgi_client(env) as client:
signup = await client.post(
"/users", data={"user": "bob"}, headers={**form, "Origin": "http://evil.test"}
)
login = await client.post(
"/login",
data={"user": "alice", "token": TOKEN},
headers={**form, "Origin": "http://evil.test"},
)
referer = await client.post(
"/login",
data={"user": "alice", "token": TOKEN},
headers={**form, "Referer": "http://evil.test/login"},
)
same_origin = await client.post(
"/login",
data={"user": "alice", "token": TOKEN},
headers={**form, "Origin": "http://gateway.test"},
)
assert signup.status_code == 403
assert login.status_code == 403
assert referer.status_code == 403
assert same_origin.status_code == 303
@pytest.mark.asyncio
async def test_auth_rate_limit_returns_429_with_retry_after(env):
configure(env, auth_rate_limit=2, auth_rate_window=60)
add_user(env.k8s)
async with asgi_client(env) as client:
first = await client.post("/login", json={"user": "alice", "token": "nope"})
second = await client.post("/login", json={"user": "alice", "token": "nope"})
third = await client.post("/login", json={"user": "alice", "token": "nope"})
assert first.status_code == 401
assert second.status_code == 401
assert third.status_code == 429
assert int(third.headers["retry-after"]) >= 1
@pytest.mark.asyncio
async def test_auth_rate_limit_is_per_client_via_rightmost_forwarded_for(env):
configure(env, auth_rate_limit=1, auth_rate_window=60)
add_user(env.k8s)
async with asgi_client(env) as client:
first = await client.post(
"/login",
json={"user": "alice", "token": "nope"},
headers={"X-Forwarded-For": "1.1.1.1, 10.0.0.1"},
)
# A spoofed leftmost entry cannot escape the same rightmost-client bucket.
same_client = await client.post(
"/login",
json={"user": "alice", "token": "nope"},
headers={"X-Forwarded-For": "9.9.9.9, 10.0.0.1"},
)
other_client = await client.post(
"/login",
json={"user": "alice", "token": "nope"},
headers={"X-Forwarded-For": "10.0.0.2"},
)
assert first.status_code == 401
assert same_client.status_code == 429
assert other_client.status_code == 401
@pytest.mark.asyncio
async def test_auth_rate_limit_zero_disables_it(env):
configure(env, auth_rate_limit=0)
add_user(env.k8s)
async with asgi_client(env) as client:
responses = [
await client.post("/login", json={"user": "alice", "token": "nope"}) for _ in range(40)
]
assert {response.status_code for response in responses} == {401}
@pytest.mark.asyncio
async def test_signup_is_rate_limited_before_the_body_is_read(env):
"""429 wins over parsing: an abusive client cannot even submit a body."""
configure(env, auth_rate_limit=1, auth_rate_window=60)
async with asgi_client(env) as client:
# Spend the single allowed auth attempt on a login, then hit signup.
await client.post("/login", json={"user": "alice", "token": "nope"})
limited = await client.post("/users", content=b"not json at all")
assert limited.status_code == 429
assert env.k8s.created == []
# --- auth ------------------------------------------------------------------
@@ -810,11 +1104,11 @@ async def test_get_user_requires_valid_bearer_or_cookie(env):
bearer = await client.get("/users/alice", headers={"Authorization": f"Bearer {TOKEN}"})
cookie = await client.get(
"/users/alice",
headers={"Cookie": f"dsh_gw_session=alice:{TOKEN}"},
headers={"Cookie": session_cookie(env)},
)
wrong_user_cookie = await client.get(
"/users/alice",
headers={"Cookie": f"dsh_gw_session=bob:{TOKEN}"},
headers={"Cookie": session_cookie(env, "bob")},
)
assert forbidden.status_code == 401
assert bearer.status_code == 200
@@ -852,7 +1146,7 @@ async def test_delete_user_requires_auth(env):
async def test_delete_with_cookie_requires_same_origin(env):
add_user(env.k8s)
env.gateway.state.handshake[("alice", POD_UID, "gateway.test")] = "dsh-auth-a=x"
cookies = {"dsh_gw_session": f"alice:{TOKEN}"}
cookies = session_dict(env)
async with asgi_client(env, cookies=cookies) as client:
missing = await client.delete("/users/alice")
cross = await client.delete("/users/alice", headers={"Origin": "http://evil.test"})
@@ -890,13 +1184,78 @@ async def test_unexpected_api_error_propagates(env):
await client.get("/users/alice")
@pytest.mark.asyncio
async def test_bearer_token_never_authorizes_another_user(env):
add_user(env.k8s, "alice", token=TOKEN)
add_user(env.k8s, "bob", token="bob-token")
async with asgi_client(env) as client:
alice_on_bob = await client.get("/users/bob", headers={"Authorization": f"Bearer {TOKEN}"})
bob_on_alice = await client.get(
"/users/alice", headers={"Authorization": "Bearer bob-token"}
)
alice_on_alice = await client.get(
"/users/alice", headers={"Authorization": f"Bearer {TOKEN}"}
)
assert alice_on_bob.status_code == 401
assert bob_on_alice.status_code == 401
assert alice_on_alice.status_code == 200
@pytest.mark.asyncio
async def test_session_cookie_never_authorizes_another_user(env):
add_user(env.k8s, "alice", token=TOKEN)
add_user(env.k8s, "bob", token="bob-token")
bob_cookie = session_cookie(env, "bob")
async with asgi_client(env) as client:
user = await client.get("/users/alice", headers={"Cookie": bob_cookie})
proxy = await client.get("/u/alice/api/state", headers={"Cookie": bob_cookie})
assert user.status_code == 401
assert proxy.status_code == 401
@pytest.mark.asyncio
async def test_opaque_session_cookie_is_not_a_bearer_token(env):
add_user(env.k8s)
_name, sid = mint_session(env)
async with asgi_client(env) as client:
response = await client.get("/users/alice", headers={"Authorization": f"Bearer {sid}"})
assert response.status_code == 401
@pytest.mark.asyncio
async def test_expired_session_is_revoked_and_treated_as_anonymous(env):
add_user(env.k8s)
name, sid = mint_session(env)
env.gateway.state.sessions[sid] = Session(user="alice", expires_at=time.time() - 1)
async with asgi_client(env, cookies={name: sid}) as client:
user = await client.get("/users/alice")
landing = await client.get("/")
assert user.status_code == 401
assert "Signed in as" not in landing.text
assert sid not in env.gateway.state.sessions
@pytest.mark.asyncio
async def test_delete_user_revokes_all_its_sessions(env):
add_user(env.k8s)
cookie = session_dict(env)
other = session_dict(env) # a second browser session for the same user
async with asgi_client(env, cookies=cookie) as client:
response = await client.delete("/users/alice", headers={"Origin": "http://gateway.test"})
assert response.status_code == 200
assert env.gateway.state.sessions == {}
async with asgi_client(env, cookies=other) as client:
landing = await client.get("/")
assert "Signed in as" not in landing.text
# --- DSH handshake ---------------------------------------------------------
@pytest.mark.asyncio
async def test_proxy_handshakes_and_uses_browser_host(env):
add_user(env.k8s)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 200
assert response.text == "hello from DSH"
@@ -912,7 +1271,7 @@ async def test_proxy_handshakes_and_uses_browser_host(env):
@pytest.mark.asyncio
async def test_dsh_cookie_is_cached_per_pod_uid_and_host(env):
add_user(env.k8s)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
first = await client.get("/u/alice/api/one")
second = await client.get("/u/alice/api/two")
assert first.status_code == second.status_code == 200
@@ -924,7 +1283,7 @@ async def test_dsh_cookie_is_cached_per_pod_uid_and_host(env):
async def test_missing_launch_token_returns_503_with_retry_after(env):
_sandbox_name, pod_name = add_user(env.k8s)
env.k8s.logs[pod_name] = "no token printed yet\n"
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
assert response.headers["retry-after"] == "10"
@@ -935,7 +1294,7 @@ async def test_missing_launch_token_returns_503_with_retry_after(env):
async def test_handshake_4xx_is_503(env):
add_user(env.k8s)
env.upstream.handshake_status = 401
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
@@ -944,11 +1303,27 @@ async def test_handshake_4xx_is_503(env):
async def test_missing_cookie_in_handshake_is_503(env):
add_user(env.k8s)
env.upstream.handshake_cookies = ["other=1; Path=/"]
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
@pytest.mark.asyncio
async def test_handshake_error_never_leaks_the_launch_token_or_url(env):
_sandbox_name, pod_name = add_user(env.k8s)
secret = "super-secret-launch-token"
env.k8s.logs[pod_name] = f"dsh web: http://10.0.0.7:3080/?token={secret}\n"
env.upstream.handshake_status = 500
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
body = response.text
assert secret not in body
assert FQDN not in body
assert "10.0.0.7" not in body
assert "dsh-auth" not in body
@pytest.mark.asyncio
async def test_pod_annotation_overrides_sandbox_name(env):
sandbox_name, pod_name = add_user(env.k8s)
@@ -957,7 +1332,7 @@ async def test_pod_annotation_overrides_sandbox_name(env):
)
env.k8s.pods["custom-pod"] = {"metadata": {"uid": "other-uid"}}
env.k8s.logs["custom-pod"] = env.k8s.logs[pod_name]
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 200
assert ("alice", "other-uid", "gateway.test") in env.gateway.state.handshake
@@ -984,7 +1359,7 @@ async def test_proxy_strips_gateway_auth_cookie_and_hop_headers(env):
env.upstream.proxy_handler = handler
headers = {
"Authorization": f"Bearer {TOKEN}",
"Cookie": "dsh_gw_session=alice:x; browser=1",
"Cookie": session_cookie(env) + "; browser=1",
"X-Forwarded-For": "1.2.3.4",
"X-Forwarded-Proto": "https",
"Host": "gateway.test",
@@ -1009,6 +1384,83 @@ async def test_proxy_strips_gateway_auth_cookie_and_hop_headers(env):
assert "connection" not in response.headers
@pytest.mark.asyncio
async def test_proxy_strips_client_supplied_forwarding_and_sso_headers(env):
add_user(env.k8s)
headers = {
"Authorization": f"Bearer {TOKEN}",
"Forwarded": "for=evil.test;proto=https",
"X-Forwarded-User": "bob",
"X-Forwarded-Host": "evil.test",
"X-Auth-Request-User": "bob",
"X-Auth-Request-Email": "bob@evil.test",
"X-Real-IP": "6.6.6.6",
"Origin": "http://gateway.test",
}
async with asgi_client(env) as client:
response = await client.get("/u/alice/api/state", headers=headers)
assert response.status_code == 200
proxy = env.upstream.proxy_requests[0]
for name in (
"forwarded",
"x-forwarded-user",
"x-auth-request-user",
"x-auth-request-email",
"x-real-ip",
):
assert name not in proxy.headers
# The gateway's own hop metadata comes from the real connection instead.
assert proxy.headers["x-forwarded-for"] == "127.0.0.1"
assert proxy.headers["x-forwarded-host"] == "gateway.test"
@pytest.mark.asyncio
async def test_proxy_upstream_header_allowlist_drops_unlisted_headers(env):
configure(env, upstream_header_allowlist=frozenset({"x-custom-ok"}))
add_user(env.k8s)
headers = {
"Authorization": f"Bearer {TOKEN}",
"X-Custom-Ok": "keep",
"X-Custom-Drop": "drop",
"X-Forwarded-User": "bob",
"Origin": "http://gateway.test",
}
async with asgi_client(env) as client:
response = await client.get("/u/alice/api/state", headers=headers)
assert response.status_code == 200
proxy = env.upstream.proxy_requests[0]
assert proxy.headers["x-custom-ok"] == "keep"
assert "x-custom-drop" not in proxy.headers
# Allowlisting never re-enables gateway-owned or spoofable headers.
assert "authorization" not in proxy.headers
assert "x-forwarded-user" not in proxy.headers
assert "cookie" not in proxy.headers or proxy.headers["cookie"] == "dsh-auth-a=one"
@pytest.mark.asyncio
async def test_proxy_rejects_requests_over_the_per_user_in_flight_cap(env):
configure(env, max_in_flight_per_user=1)
add_user(env.k8s)
env.gateway.state.in_flight["alice"] = 1
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
assert "retry-after" in response.headers
assert env.gateway.state.in_flight["alice"] == 1
assert env.upstream.proxy_requests == []
@pytest.mark.asyncio
async def test_proxy_accepts_requests_below_the_per_user_in_flight_cap(env):
configure(env, max_in_flight_per_user=2)
add_user(env.k8s)
env.gateway.state.in_flight["alice"] = 1
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 200
assert env.gateway.state.in_flight["alice"] == 1 # the slot was released again
@pytest.mark.asyncio
async def test_proxy_rewrites_absolute_path_location(env):
add_user(env.k8s)
@@ -1019,7 +1471,7 @@ async def test_proxy_rewrites_absolute_path_location(env):
("set-cookie", "upstream=leak; Path=/"),
],
)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/redirect", follow_redirects=False)
assert response.status_code == 303
assert response.headers["location"] == "/u/alice/?next=1"
@@ -1040,7 +1492,7 @@ async def test_proxy_refreshes_cookie_once_on_upstream_401(env):
return dsh_response(200, b"fresh")
env.upstream.proxy_handler = handler
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 200
assert response.text == "fresh"
@@ -1053,7 +1505,7 @@ async def test_proxy_refreshes_cookie_once_on_upstream_401(env):
async def test_proxy_does_not_refresh_401_without_cached_cookie(env):
add_user(env.k8s)
env.upstream.proxy_handler = lambda request: dsh_response(401, b"no")
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 401
assert len(env.upstream.proxy_requests) == 1
@@ -1067,7 +1519,7 @@ async def test_proxy_upstream_connection_error_is_502(env):
raise httpx.ConnectError("boom", request=request)
env.upstream.proxy_handler = handler
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 502
assert env.gateway.state.in_flight["alice"] == 0
@@ -1106,7 +1558,7 @@ async def test_proxy_client_disconnect_mid_body_returns_499(env):
"/u/alice/api/echo",
headers=[
(b"origin", b"http://gateway.test"),
(b"cookie", f"dsh_gw_session=alice:{TOKEN}".encode()),
(b"cookie", session_cookie(env).encode()),
(b"content-type", b"application/json"),
],
),
@@ -1167,7 +1619,7 @@ async def test_proxy_client_disconnect_mid_stream_frees_slot(env):
raw_scope(
"GET",
"/u/alice/api/stream",
headers=[(b"cookie", f"dsh_gw_session=alice:{TOKEN}".encode())],
headers=[(b"cookie", session_cookie(env).encode())],
),
receive,
)
@@ -1234,7 +1686,7 @@ async def test_proxy_client_disconnect_survives_cancelled_upstream_close(env):
raw_scope(
"GET",
"/u/alice/api/stream",
headers=[(b"cookie", f"dsh_gw_session=alice:{TOKEN}".encode())],
headers=[(b"cookie", session_cookie(env).encode())],
),
receive,
)
@@ -1258,7 +1710,7 @@ async def test_proxy_wakes_suspended_sandbox(env, monkeypatch):
return env.k8s.sandboxes[sandbox_name]
monkeypatch.setattr(env.gateway, "wait_ready", ready)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 200
assert ("dsh-alice-sbx", "Running") in env.k8s.patches
@@ -1273,7 +1725,7 @@ async def test_proxy_wake_timeout_returns_503(env, monkeypatch):
return None
monkeypatch.setattr(env.gateway, "wait_ready", never)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/u/alice/api/state")
assert response.status_code == 503
assert response.headers["retry-after"] == "10"
@@ -1297,7 +1749,7 @@ async def test_proxy_requires_auth(env):
@pytest.mark.asyncio
async def test_proxy_cookie_mutation_requires_same_origin(env):
add_user(env.k8s)
cookies = {"dsh_gw_session": f"alice:{TOKEN}"}
cookies = session_dict(env)
async with asgi_client(env, cookies=cookies) as client:
cross = await client.post(
"/u/alice/api/mutate", headers={"Origin": "http://evil.test"}, content=b"{}"
@@ -1385,7 +1837,7 @@ def test_ws_cookie_cross_origin_closes_4401(env):
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={
"cookie": f"dsh_gw_session=alice:{TOKEN}",
"cookie": session_cookie(env),
"origin": "http://evil.test",
},
) as ws:
@@ -1425,7 +1877,7 @@ def test_ws_cookie_same_origin_bridges_frames(env):
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={
"cookie": f"dsh_gw_session=alice:{TOKEN}",
"cookie": session_cookie(env),
"origin": "http://testserver",
},
) as ws:
@@ -1464,6 +1916,70 @@ def test_ws_upstream_connect_failure_closes_1011(env):
assert exc.value.code == 1011
def test_ws_strips_client_supplied_forwarding_and_sso_headers(env):
add_user(env.k8s)
with TestClient(env.app) as client:
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={
"Authorization": f"Bearer {TOKEN}",
"Forwarded": "for=evil.test",
"X-Forwarded-User": "bob",
"X-Auth-Request-User": "bob",
},
) as ws:
ws.send_text("hello")
assert ws.receive_text() == "welcome"
assert ws.receive_text() == "pong"
headers = env.ws_connect.calls[0][1]["additional_headers"]
assert "forwarded" not in headers
assert "x-forwarded-user" not in headers
assert "x-auth-request-user" not in headers
assert headers["cookie"] == "dsh-auth-a=one"
def test_ws_per_user_in_flight_cap_closes_1011(env):
configure(env, max_in_flight_per_user=1)
add_user(env.k8s)
env.gateway.state.in_flight["alice"] = 1
with TestClient(env.app) as client:
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={"Authorization": f"Bearer {TOKEN}"},
) as ws:
with pytest.raises(WebSocketDisconnect) as exc:
ws.receive_text()
assert exc.value.code == 1011
assert env.ws_connect.calls == []
def test_ws_passes_the_default_max_message_size(env):
add_user(env.k8s)
with TestClient(env.app) as client:
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={"Authorization": f"Bearer {TOKEN}"},
) as ws:
ws.send_text("hello")
assert ws.receive_text() == "welcome"
assert ws.receive_text() == "pong"
assert env.ws_connect.calls[0][1]["max_size"] == 16 * 1024 * 1024
def test_ws_max_message_size_is_configurable(env):
configure(env, ws_max_message_bytes=2048)
add_user(env.k8s)
with TestClient(env.app) as client:
with client.websocket_connect(
"/u/alice/api/remote.mux",
headers={"Authorization": f"Bearer {TOKEN}"},
) as ws:
ws.send_text("hello")
assert ws.receive_text() == "welcome"
assert ws.receive_text() == "pong"
assert env.ws_connect.calls[0][1]["max_size"] == 2048
# --- idle sweeper ----------------------------------------------------------
@@ -1576,14 +2092,46 @@ async def test_sweeper_survives_transient_api_error(env):
assert env.gateway.state.maintenance["alice"] is False
@pytest.mark.asyncio
async def test_sweeper_prunes_orphan_and_expired_state(env):
now = time.time()
# No claim for "ghost": every per-user entry is orphaned.
env.gateway.state.last_activity["ghost"] = now
env.gateway.state.in_flight["ghost"] = 0
env.gateway.state.maintenance["ghost"] = False
env.gateway.state.handshake[("ghost", POD_UID, "gateway.test")] = "dsh-auth-a=x"
_ghost_name, ghost_sid = mint_session(env, "ghost")
env.gateway.state.auth_rate["1.2.3.4"] = (now - 3600, 5)
# A user with requests still in flight is preserved even without a claim.
env.gateway.state.in_flight["busy"] = 2
# A live claim keeps its user's current session...
env.k8s.claims["alice"] = ready_claim()
env.k8s.sandboxes["dsh-alice-sbx"] = ready_sandbox()
env.gateway.state.last_activity["alice"] = now
_alice_name, alice_sid = mint_session(env, "alice")
# ...but an expired session is pruned even for a live user.
_old_name, expired_sid = mint_session(env, "alice")
env.gateway.state.sessions[expired_sid] = Session(user="alice", expires_at=now - 1)
await env.gateway.sweep_once()
assert "ghost" not in env.gateway.state.last_activity
assert "ghost" not in env.gateway.state.in_flight
assert "ghost" not in env.gateway.state.maintenance
assert ("ghost", POD_UID, "gateway.test") not in env.gateway.state.handshake
assert ghost_sid not in env.gateway.state.sessions
assert env.gateway.state.auth_rate == {}
assert env.gateway.state.in_flight["busy"] == 2
assert alice_sid in env.gateway.state.sessions
assert expired_sid not in env.gateway.state.sessions
# --- SSO (reverse-proxy identity) ------------------------------------------
def enable_sso(env, header="X-Forwarded-User"):
"""Flip an env into SSO mode: the gateway trusts the named identity header."""
env.settings = Settings(**{**env.settings.__dict__, "auth_proxy_header": header})
env.gateway.settings = env.settings
return env
return configure(env, auth_proxy_header=header)
def test_remote_user_none_when_sso_disabled(env):
@@ -1611,7 +2159,7 @@ async def test_sso_landing_new_user_shows_provisioning(env):
assert "Signed in as <strong>alice</strong>" in response.text
assert "Provisioning" in response.text
assert "Recreate sandbox" not in response.text
assert re_cookie_user(response.headers["set-cookie"]) == "alice"
assert response_session_user(env, response) == "alice"
assert [c[0] for c in env.k8s.created] == ["alice"]
assert env.k8s.token_patches == []
@@ -1620,7 +2168,7 @@ async def test_sso_landing_new_user_shows_provisioning(env):
async def test_sso_landing_ready_connects_and_offers_recreate_only(env):
enable_sso(env)
add_user(env.k8s)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.get("/", headers={"X-Forwarded-User": "alice"})
assert response.status_code == 200
assert "Signed in as <strong>alice</strong>" in response.text
@@ -1642,11 +2190,55 @@ async def test_sso_landing_rotates_token_when_session_missing(env):
response = await client.get("/", headers={"X-Forwarded-User": "alice"})
assert response.status_code == 200
assert "Recreate sandbox" in response.text
assert re_cookie_user(response.headers["set-cookie"]) == "alice"
assert response_session_user(env, response) == "alice"
assert [p[0] for p in env.k8s.token_patches] == ["alice"]
assert env.k8s.created == []
@pytest.mark.asyncio
async def test_sso_landing_reuses_a_valid_session_for_the_remote_user(env):
enable_sso(env)
add_user(env.k8s)
name, sid = mint_session(env)
async with asgi_client(env, cookies={name: sid}) as client:
response = await client.get("/", headers={"X-Forwarded-User": "alice"})
assert response.status_code == 200
assert "Recreate sandbox" in response.text
# Untouched: no cookie is re-issued and no token is rotated.
assert "set-cookie" not in response.headers
assert [p[0] for p in env.k8s.token_patches] == []
assert env.gateway.state.sessions[sid].user == "alice"
@pytest.mark.asyncio
async def test_sso_landing_revokes_a_foreign_session(env):
enable_sso(env)
add_user(env.k8s)
bob_name, bob_sid = mint_session(env, "bob")
async with asgi_client(env, cookies={bob_name: bob_sid}) as client:
response = await client.get("/", headers={"X-Forwarded-User": "alice"})
assert response.status_code == 200
# A session for another identity is never reused and is revoked server-side.
assert bob_sid not in env.gateway.state.sessions
assert response_session_user(env, response) == "alice"
@pytest.mark.asyncio
async def test_recreate_revokes_the_old_session_and_mints_a_new_one(env):
enable_sso(env)
add_user(env.k8s)
name, sid = mint_session(env)
async with asgi_client(env, cookies={name: sid}) as client:
response = await client.post(
"/recreate",
headers={"X-Forwarded-User": "alice", "Origin": "http://gateway.test"},
)
assert response.status_code == 303
assert sid not in env.gateway.state.sessions
new_sid = response.headers["set-cookie"].split(";", 1)[0].split("=", 1)[1]
assert env.gateway.state.sessions[new_sid].user == "alice"
@pytest.mark.asyncio
async def test_recreate_deletes_and_reprovisions(env):
enable_sso(env)
@@ -1657,7 +2249,7 @@ async def test_recreate_deletes_and_reprovisions(env):
assert response.headers["location"] == "/u/alice/"
assert env.k8s.deleted == ["alice"]
assert [c[0] for c in env.k8s.created] == ["alice"]
assert re_cookie_user(response.headers["set-cookie"]) == "alice"
assert response_session_user(env, response) == "alice"
@pytest.mark.asyncio
@@ -1673,7 +2265,7 @@ async def test_recreate_requires_sso(env):
async def test_recreate_rejects_cross_origin(env):
enable_sso(env)
add_user(env.k8s)
async with asgi_client(env, cookies={"dsh_gw_session": f"alice:{TOKEN}"}) as client:
async with asgi_client(env, cookies=session_dict(env)) as client:
response = await client.post(
"/recreate",
headers={"X-Forwarded-User": "alice", "Origin": "http://evil.test"},

18
examples/dsh-agents-as-a-service/manifests/10-sandboxtemplate.yaml
View File

@@ -131,6 +131,24 @@ spec:
# a private PVC), so trade DSH's inner file policy for a working
# agent. Set this to workspace-write (or read-only) on hosts where a
# backend is available; see the README troubleshooting row.
# SECURITY: the pod is the isolation boundary in this example.
# danger-full-access is set only because DSH's own command sandbox
# has no usable backend in this pod (see above), which means an
# agent command can read its own DSH_HOME (including
# .credentials.yaml) and write anywhere in the container. What
# still contains it is the pod itself: one pod per user, non-root,
# all capabilities dropped, RuntimeDefault seccomp, its own PVC and
# Service. That is NOT a hardened multi-tenant boundary on a shared
# kernel. For production/multi-tenant use, additionally run the
# sandbox under gVisor (`runtimeClassName: gvisor` above) so a
# container escape is trapped in a userspace kernel, and apply the
# sandbox NetworkPolicy (manifests/optional/network-policy/) so
# tenants cannot reach each other or cluster-internal endpoints.
# The base manifests without those two controls are a demo. Treat
# danger-full-access as a fallback, not a target: on hosts where
# DSH's command-sandbox backend works, prefer workspace-write (or
# read-only), and recreate the pod because the value is stamped at
# creation.
- name: DSH_PERMISSION_MODE
value: danger-full-access
# Probes target /favicon.svg: DSH gates / behind its launch-token

68
examples/dsh-agents-as-a-service/manifests/50-gateway.yaml
View File

@@ -134,14 +134,30 @@ spec:
# the DSH launch token.
- name: DSH_CONTAINER
value: dsh-web
# Cookie dsh_gw_session is HttpOnly + SameSite=Lax. "false" keeps
# the plain-HTTP demo usable through port-forward: a Secure cookie
# is never sent over HTTP, so login would silently fail.
# HTTPS deployments (the ingress/ variant with TLS, or
# gateway-api-sso/, which terminates TLS at the Gateway) MUST set
# this to "true".
# Browser session cookie: HttpOnly + SameSite=Lax, holds only an
# opaque, server-side session id (sid) - never the user's DSH
# token. Sessions can be revoked server-side: POST /logout, user
# delete, POST /recreate and expiry (COOKIE_MAX_AGE, default
# 7 days) all invalidate them. The session table is in process
# memory, so restarting this Deployment logs every browser out.
# With COOKIE_SECURE=true the cookie is named
# __Host-dsh_gw_session; COOKIE_NAME overrides the name.
#
# "false" keeps the plain-HTTP demo usable through port-forward: a
# Secure cookie is never sent over HTTP, so login would silently
# fail. The gateway REFUSES TO START with COOKIE_SECURE=false
# unless ALLOW_INSECURE_COOKIES=1 acknowledges that the session
# travels in clear text, so the base demo sets both. HTTPS
# deployments (the ingress/ variant with TLS, or gateway-api-sso/,
# which terminates TLS at the Gateway) MUST set this to "true".
- name: COOKIE_SECURE
value: "false"
# Explicit acknowledgement that this base demo serves plain HTTP.
# Required when COOKIE_SECURE=false (the gateway exits at startup
# otherwise); ignored once COOKIE_SECURE=true, so HTTPS installs
# should drop it. Never set this on an internet-facing deployment.
- name: ALLOW_INSECURE_COOKIES
value: "1"
# SSO IS OPT-IN AND DISABLED HERE. With AUTH_PROXY_HEADER unset the
# gateway trusts only its own bearer/session auth and ignores every
# identity header. Enable it ONLY behind
@@ -152,9 +168,49 @@ spec:
# reach :8080 can impersonate any user by sending that header.
# - name: AUTH_PROXY_HEADER
# value: X-Forwarded-User
# Brute-force / abuse throttle for the unauthenticated auth
# endpoints (/users and /login): at most N attempts per client per
# AUTH_RATE_WINDOW seconds (default 60). A client is the rightmost
# X-Forwarded-For entry (the one added by the trusted proxy) when
# present, otherwise the TCP peer. Over the limit the gateway
# answers 429 + Retry-After BEFORE reading the body or touching the
# cluster; 0 disables the limiter. This is a per-process bucket, so
# it is not a substitute for an ingress rate limit.
- name: AUTH_RATE_LIMIT
value: "30"
# Window in seconds for AUTH_RATE_LIMIT (default 60).
# - name: AUTH_RATE_WINDOW
# value: "60"
# Optional signup gate. When set, POST /users requires the invite
# code - form field "invite" or header X-Signup-Invite - compared in
# constant time. A missing/wrong code is rejected with 403 before
# the user is looked up, so it also stops unauthenticated claim
# creation. The landing page shows the invite input only when this
# is set. Keep the value in a Secret, create it before rollout and
# restart the gateway to pick up a rotation:
# kubectl -n dsh-demo create secret generic dsh-signup-invite \
# --from-literal=code="$(openssl rand -hex 16)"
# - name: SIGNUP_INVITE_CODE
# valueFrom:
# secretKeyRef:
# name: dsh-signup-invite
# key: code
# Reject proxied request bodies larger than this (413).
- name: MAX_BODY_MB
value: "32"
# Per-user concurrency guard (default 32): at most this many
# proxied HTTP requests and WebSocket connections per user.
# Excess HTTP requests get 503 + Retry-After and excess WebSocket
# upgrades close with 1011, so one tenant cannot exhaust the
# single-replica gateway for everyone else.
# - name: MAX_IN_FLIGHT_PER_USER
# value: "32"
# Largest WebSocket message received from DSH, in bytes (default
# 16 MiB); oversized frames close the connection. Browser-to-DSH
# frames are bounded by uvicorn's own limit. HTTP/SSE responses are
# streamed without this cap.
# - name: WS_MAX_MESSAGE_BYTES
# value: "16777216"
# /healthz is unauthenticated and returns 200 without touching the
# cluster, so it is safe for both readiness and liveness.
readinessProbe:

43
examples/dsh-agents-as-a-service/manifests/exposure/README.md
View File

@@ -16,6 +16,16 @@ Applying both gives the gateway two entry points, and the SSO variant is only
safe when the gateway is reachable **exclusively** through oauth2-proxy. Do not
mix them.
> **Production requirement.** Each variant's NetworkPolicy is part of the
> variant, and applying the variant without it is demo-only. The sandbox
> NetworkPolicy and the gateway egress policy (the API server is cluster-specific)
> are separate and equally required; see the
> [production checklist](../../docs/optional-hardening.md#6-production-checklist)
> and the
> [gateway egress recipe](../../docs/optional-hardening.md#gateway-egress-required-for-production).
> For SSO, run [`scripts/preflight-sso.sh`](../../scripts/preflight-sso.sh)
> after applying the variant and before sending real traffic.
Each variant also ships a NetworkPolicy, applied by the same directory apply:
- `ingress/61-networkpolicy.yaml` allows only Traefik pods to reach the gateway
@@ -27,7 +37,8 @@ Each variant also ships a NetworkPolicy, applied by the same directory apply:
The policies assume Traefik's `app.kubernetes.io/name: traefik` pod label.
Adjust the selector for another implementation or a `hostNetwork` install; see
[optional hardening](../../docs/optional-hardening.md) for the probe caveat and
verification commands.
verification commands. Keep the policies: an exposure without its NetworkPolicy
is demo-only.
## Applying a variant
@@ -69,10 +80,13 @@ Secrets imperatively (commands below) before applying the directory.
2. Set `DSH_TRUSTED_HOSTS` in the SandboxTemplate to the same public host:
the gateway forwards the browser's `Host`, and DSH rejects unknown hosts on
`/api` (see `../10-sandboxtemplate.yaml`).
3. Set `COOKIE_SECURE=true` on the gateway once the Ingress serves HTTPS:
`kubectl -n dsh-demo set env deploy/dsh-gateway COOKIE_SECURE=true`. The
base manifest keeps `false` so the plain-HTTP port-forward demo works; a
`Secure` cookie is never sent over HTTP, which silently breaks login.
3. Set `COOKIE_SECURE=true` and remove the plain-HTTP acknowledgement once the
Ingress serves HTTPS:
`kubectl -n dsh-demo set env deploy/dsh-gateway COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-`.
The base manifest keeps `false` + `ALLOW_INSECURE_COOKIES=1` so the
plain-HTTP port-forward demo works; a `Secure` cookie is never sent over HTTP,
which silently breaks login, while the acknowledgement must not survive into
production.
4. Do not attach buffering or compression middlewares to this router: they
break WebSocket upgrades and streaming.
5. `61-networkpolicy.yaml` is part of this variant: it allows only Traefik pods
@@ -116,14 +130,23 @@ kubectl -n dsh-demo create secret tls dsh-demo-tls --cert=tls.crt --key=tls.key
# 3. Enable SSO on the gateway and trust the public host in the template.
kubectl -n dsh-demo set env deploy/dsh-gateway \
AUTH_PROXY_HEADER=X-Forwarded-User COOKIE_SECURE=true
AUTH_PROXY_HEADER=X-Forwarded-User COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-
# then set DSH_TRUSTED_HOSTS=dsh-demo.example.com in the SandboxTemplate
# (see ../10-sandboxtemplate.yaml) and re-apply it.
# 4. Apply the variant.
kubectl apply -f manifests/exposure/gateway-api-sso/
# 5. Preflight: fails loudly on missing Secrets, a bad oauth2-proxy cookie
# secret, missing gateway SSO env, a dangling TLS reference, leftover
# placeholders, or a second path to the gateway.
./scripts/preflight-sso.sh
```
The preflight is read-only and exits non-zero on any failure, naming the exact
fix. Run it again after rotating a Secret or changing the public host; it also
verifies that the oauth2-proxy cookie secret decodes to 16, 24 or 32 bytes.
### SSO trust boundary
`AUTH_PROXY_HEADER` is opt-in and deliberately unset in the base gateway
@@ -141,8 +164,14 @@ takes all of these:
impersonate any user.
3. Keep `75-networkpolicies.yaml` with the variant. It enforces the boundary in
the network layer (gateway ← oauth2-proxy ← Traefik, and limited oauth2-proxy
egress). A custom exposure without an equivalent policy has no network
egress). This is required, not optional: an exposure without the policy is
demo-only, and a custom exposure without an equivalent policy has no network
enforcement.
4. Apply the gateway egress policy too: the gateway talks to the Kubernetes API
and to sandbox pods, and `75-networkpolicies.yaml` intentionally leaves that
direction unrestricted because the API-server endpoint is cluster-specific.
See
[gateway egress](../../docs/optional-hardening.md#gateway-egress-required-for-production).
Also terminate TLS in front of the gateway (the Gateway listener does) and
leave `COOKIE_SECURE=true` so the session cookie never travels in clear text.

7
examples/dsh-agents-as-a-service/manifests/exposure/gateway-api-sso/72-oauth2-proxy.yaml
View File

@@ -25,8 +25,9 @@
# TLS terminates at the Gateway listener (70-gateway.yaml); keep
# --cookie-secure=true and COOKIE_SECURE=true on the gateway.
#
# The image is pinned to quay.io/oauth2-proxy/oauth2-proxy:v7.15.5
# (versions.env: OAUTH2_PROXY_IMAGE). Credentials come from the
# The image is digest-pinned: quay.io/oauth2-proxy/oauth2-proxy:v7.15.5@sha256:...
# (versions.env: OAUTH2_PROXY_IMAGE). The digest is the immutable multi-arch
# index, so the tag can never be moved under us. Credentials come from the
# "oauth2-proxy" Secret (see 73-oauth2-proxy-secret.yaml.example); this
# manifest never carries secrets.
apiVersion: apps/v1
@@ -61,7 +62,7 @@ spec:
type: RuntimeDefault
containers:
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.5
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.5@sha256:8498b0d0ef0a7b29686414000a08aee467f02d0299c9ed1e006a8f33fc017916
imagePullPolicy: IfNotPresent
securityContext:
allowPrivilegeEscalation: false

9
examples/dsh-agents-as-a-service/manifests/exposure/gateway-api-sso/75-networkpolicies.yaml
View File

@@ -1,5 +1,5 @@
# OPTIONAL with the SSO exposure variant: enforce the trust boundary at the
# network layer.
# REQUIRED with the SSO exposure variant: enforce the trust boundary at the
# network layer. Applying the SSO variant without this file is demo-only.
#
# The chain is Traefik (Gateway API) -> oauth2-proxy -> dsh-gateway. These
# policies make it the ONLY chain:
@@ -8,8 +8,9 @@
# - oauth2-proxy egress is limited to DNS, public HTTPS (GitHub) and the
# gateway.
# The gateway's own egress (Kubernetes API, sandbox pods) is intentionally not
# restricted here: the API-server endpoint is cluster-specific. See
# ../../../docs/optional-hardening.md for that recipe.
# restricted here: the API-server endpoint is cluster-specific. Add it for
# production. See
# ../../../docs/optional-hardening.md#gateway-egress-required-for-production.
#
# This manifest assumes Traefik, matching gatewayClassName: traefik in
# 70-gateway.yaml. Adjust the app.kubernetes.io/name selector for another

17
examples/dsh-agents-as-a-service/manifests/exposure/ingress/60-ingress.yaml
View File

@@ -14,8 +14,13 @@
# DSH_TRUSTED_HOSTS to this host so DSH accepts the forwarded Host/Origin
# on /api.
# 2. Gateway Deployment (manifests/50-gateway.yaml): set COOKIE_SECURE to
# "true" once this Ingress serves HTTPS, so the session cookie is Secure.
# The base manifest sets "false" for the plain-HTTP port-forward demo.
# "true" and remove ALLOW_INSECURE_COOKIES once this Ingress serves HTTPS,
# so the session cookie is Secure:
# kubectl -n dsh-demo set env deploy/dsh-gateway \
# COOKIE_SECURE=true ALLOW_INSECURE_COOKIES-
# The base manifest sets "false" + ALLOW_INSECURE_COOKIES=1 for the
# plain-HTTP port-forward demo; without the acknowledgement the gateway
# refuses to start.
#
# WebSocket support is REQUIRED: the browser talks to /u/<user>/... and DSH
# uses WebSockets for live session traffic. Traefik upgrades WS/SSE
@@ -24,9 +29,11 @@
# and upgrades. `path: /` Prefix covers the landing page and every per-user
# proxy path, so no per-sandbox Ingress is needed.
#
# 61-networkpolicy.yaml is part of this variant: it restricts the gateway's
# ingress to Traefik pods. Adjust its selector if your ingress controller uses
# a different label or hostNetwork.
# 61-networkpolicy.yaml is part of this variant and required for production:
# it restricts the gateway's ingress to Traefik pods. Adjust its selector if
# your ingress controller uses a different label or hostNetwork. The sandbox
# and gateway-egress NetworkPolicies are separate requirements; see
# ../../../docs/optional-hardening.md#6-production-checklist.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:

9
examples/dsh-agents-as-a-service/manifests/exposure/ingress/61-networkpolicy.yaml
View File

@@ -1,11 +1,16 @@
# OPTIONAL with the Ingress exposure variant: restrict the gateway's ingress
# to the ingress controller.
# REQUIRED with the Ingress exposure variant for production: restrict the
# gateway's ingress to the ingress controller. Applying the variant without
# this policy is demo-only.
#
# This manifest assumes Traefik, matching this variant's
# `ingressClassName: traefik`. Traefik's Helm chart labels its pods
# app.kubernetes.io/name=traefik; adjust the selector for another controller or
# for a hostNetwork install (where the source is the node IP, not a pod IP).
#
# The sandbox NetworkPolicy and the gateway egress policy are separate and also
# required for production; see
# ../../../docs/optional-hardening.md#6-production-checklist.
#
# NetworkPolicy enforcement of kubelet probe traffic is CNI-dependent. If the
# gateway starts failing its /healthz probes, allow the node CIDR on port 8080
# or remove this policy. See ../../../docs/optional-hardening.md.

6
examples/dsh-agents-as-a-service/manifests/optional/README.md
View File

@@ -23,6 +23,12 @@ It requires a CNI that enforces `NetworkPolicy`. The selector uses the
agent-sandbox controller's pod tracking label, so the policy applies to
existing warm-pool and claimed sandboxes without recreation.
**This is opt-in for the demo but required for production.** Without it every
tenant can reach every other tenant and cluster-internal endpoints. Production
also needs the gateway egress policy (cluster-specific API-server endpoint) and
the exposure variant's policy; see the
[production checklist](../../docs/optional-hardening.md#6-production-checklist).
## gVisor, nodeSelector and provider keys
These are configured in the main manifests or with an extra Secret:

8
examples/dsh-agents-as-a-service/manifests/optional/network-policy/10-sandbox-networkpolicy.yaml
View File

@@ -1,9 +1,15 @@
# OPTIONAL, opt-in tenant network policy for the DSH sandboxes.
# OPTIONAL for the base walkthrough, REQUIRED for production. Opt-in tenant
# network policy for the DSH sandboxes: without it every tenant can reach every
# other tenant and cluster-internal endpoints.
#
# It is NOT part of the top-level walkthrough. Apply and delete it explicitly:
# kubectl apply -f manifests/optional/network-policy/
# kubectl delete -f manifests/optional/network-policy/
#
# Production also needs the gateway egress policy (cluster-specific API-server
# endpoint) and the exposure variant's policy, plus gVisor
# (docs/optional-hardening.md#6-production-checklist).
#
# The selector uses the agent-sandbox controller's pod tracking label, which is
# present on every Sandbox pod (including warm-pool spares), so the policy
# applies to existing sandboxes without recreating them. The SandboxTemplate

234
examples/dsh-agents-as-a-service/scripts/preflight-sso.sh Executable file
View File

@@ -0,0 +1,234 @@
#!/usr/bin/env bash
# Preflight for the gateway-api-sso exposure variant.
#
# Run this AFTER creating the oauth2-proxy/TLS Secrets, setting the gateway
# env and applying manifests/exposure/gateway-api-sso/, and BEFORE sending real
# traffic. It checks the configuration that is easy to get wrong and whose
# failure is either silent (login appears to work but the gateway still asks
# for its own token) or dangerous (a spoofable identity header, a placeholder
# host, a TLS Secret that does not exist).
#
# Checks:
# 1. kubectl reaches the cluster and the SSO objects exist;
# 2. the oauth2-proxy Secret has client-id, client-secret and a cookie-secret
# that decodes to 16, 24 or 32 bytes; the TLS Secret exists;
# 3. deploy/dsh-gateway has AUTH_PROXY_HEADER=X-Forwarded-User and
# COOKIE_SECURE=true (both are opt-in and both are required for SSO over
# TLS);
# 4. the Gateway's TLS certificate reference resolves to an existing Secret;
# 5. no your-github-org / dsh-demo.example.com placeholder is left in the
# applied oauth2-proxy args, Gateway listeners or HTTPRoute hostnames;
# 6. (warning) no Ingress in the namespace creates a second path to
# dsh-gateway. With SSO enabled the Service must be reachable only through
# oauth2-proxy.
#
# Exit status: 0 when every check passes (warnings allowed), 1 on any failure.
# Every failure prints the fix. This script only reads cluster state.
#
# Usage:
# examples/dsh-agents-as-a-service/scripts/preflight-sso.sh
#
# Environment overrides:
# NAMESPACE namespace (default: dsh-demo)
# GATEWAY_DEPLOY gateway Deployment name (default: dsh-gateway)
# OAUTH_DEPLOY oauth2-proxy Deployment name (default: oauth2-proxy)
# OAUTH_SECRET oauth2-proxy Secret name (default: oauth2-proxy)
# TLS_SECRET TLS Secret name the Gateway should reference (default:
# dsh-demo-tls)
set -euo pipefail
cd "$(dirname "$0")/.."
NS="${NAMESPACE:-dsh-demo}"
GATEWAY_DEPLOY="${GATEWAY_DEPLOY:-dsh-gateway}"
OAUTH_DEPLOY="${OAUTH_DEPLOY:-oauth2-proxy}"
OAUTH_SECRET="${OAUTH_SECRET:-oauth2-proxy}"
TLS_SECRET="${TLS_SECRET:-dsh-demo-tls}"
FAILED=0
pass() { printf 'PASS %s\n' "$*"; }
warn() { printf 'WARN %s\n' "$*" >&2; }
fail() { printf 'FAIL %s\n' "$*" >&2; FAILED=1; }
if ! command -v kubectl >/dev/null 2>&1; then
echo "FAIL kubectl is not installed; install it and select the target context" >&2
exit 1
fi
if ! kubectl -n "$NS" get deployment "$GATEWAY_DEPLOY" >/dev/null 2>&1; then
echo "FAIL deploy/$GATEWAY_DEPLOY not found in namespace $NS." >&2
echo " Apply the base manifests first: kubectl apply -f manifests/" >&2
echo " Then apply the variant: kubectl apply -f manifests/exposure/gateway-api-sso/" >&2
exit 1
fi
echo "== Secrets"
# check_secret_key <secret> <key> <hint>
check_secret_key() {
local secret=$1 key=$2 hint=$3 value escaped
# Kubernetes JSONPath treats "." as a path separator; escape it in keys such
# as tls.crt.
escaped=${key//./\\.}
value=$(kubectl -n "$NS" get secret "$secret" -o jsonpath="{.data.${escaped}}" 2>/dev/null || true)
if [ -n "$value" ]; then
pass "Secret $secret has non-empty key '$key'"
else
fail "Secret $secret is missing a non-empty '$key' key. $hint"
fi
}
check_secret_key "$OAUTH_SECRET" client-id \
"Create it from manifests/exposure/gateway-api-sso/73-oauth2-proxy-secret.yaml.example."
check_secret_key "$OAUTH_SECRET" client-secret \
"Create it from manifests/exposure/gateway-api-sso/73-oauth2-proxy-secret.yaml.example."
check_secret_key "$OAUTH_SECRET" cookie-secret \
"Create it from manifests/exposure/gateway-api-sso/73-oauth2-proxy-secret.yaml.example."
# The cookie secret must be URL-safe unpadded base64 of 16, 24 or 32 bytes;
# oauth2-proxy refuses anything else (the classic error is "44 bytes").
if COOKIE_B64=$(kubectl -n "$NS" get secret "$OAUTH_SECRET" \
-o jsonpath='{.data.cookie-secret}' 2>/dev/null); then
if [ -n "$COOKIE_B64" ]; then
if ! COOKIE_LEN=$(printf '%s' "$COOKIE_B64" | base64 -d 2>/dev/null | wc -c); then
fail "Secret $OAUTH_SECRET key 'cookie-secret' is not valid base64. Generate it with: python3 -c 'import secrets; print(secrets.token_urlsafe(32))'"
else
case "$COOKIE_LEN" in
16 | 24 | 32) pass "oauth2-proxy cookie-secret decodes to $COOKIE_LEN bytes" ;;
*)
fail "oauth2-proxy cookie-secret decodes to $COOKIE_LEN bytes; it must be 16, 24 or 32. Generate a URL-safe unpadded value with: python3 -c 'import secrets; print(secrets.token_urlsafe(32))'"
;;
esac
fi
fi
fi
check_secret_key "$TLS_SECRET" tls.crt \
"Create the TLS Secret with: kubectl -n $NS create secret tls $TLS_SECRET --cert=tls.crt --key=tls.key"
check_secret_key "$TLS_SECRET" tls.key \
"Create the TLS Secret with: kubectl -n $NS create secret tls $TLS_SECRET --cert=tls.crt --key=tls.key"
echo
echo "== Gateway SSO configuration (deploy/$GATEWAY_DEPLOY)"
GATEWAY_ENV=$(kubectl -n "$NS" get deployment "$GATEWAY_DEPLOY" \
-o jsonpath='{range .spec.template.spec.containers[*].env[*]}{.name}={.value}{"\n"}{end}')
# env_value <NAME>: print the literal env value, or nothing when unset.
env_value() {
printf '%s\n' "$GATEWAY_ENV" | awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print }'
}
env_is_set() {
printf '%s\n' "$GATEWAY_ENV" | awk -F= -v key="$1" '$1 == key { found = 1 } END { exit found ? 0 : 1 }'
}
if ! env_is_set AUTH_PROXY_HEADER; then
fail "AUTH_PROXY_HEADER is not set on deploy/$GATEWAY_DEPLOY, so the gateway ignores oauth2-proxy's identity header and users are still asked for the gateway token. Fix: kubectl -n $NS set env deploy/$GATEWAY_DEPLOY AUTH_PROXY_HEADER=X-Forwarded-User"
elif [ "$(env_value AUTH_PROXY_HEADER)" != "X-Forwarded-User" ]; then
fail "AUTH_PROXY_HEADER is '$(env_value AUTH_PROXY_HEADER)', expected 'X-Forwarded-User'. Fix: kubectl -n $NS set env deploy/$GATEWAY_DEPLOY AUTH_PROXY_HEADER=X-Forwarded-User"
else
pass "AUTH_PROXY_HEADER=X-Forwarded-User"
fi
if ! env_is_set COOKIE_SECURE; then
fail "COOKIE_SECURE is not set on deploy/$GATEWAY_DEPLOY; the session cookie must be Secure behind TLS. Fix: kubectl -n $NS set env deploy/$GATEWAY_DEPLOY COOKIE_SECURE=true"
elif [ "$(env_value COOKIE_SECURE | tr '[:upper:]' '[:lower:]')" != "true" ]; then
fail "COOKIE_SECURE is '$(env_value COOKIE_SECURE)', expected 'true' behind TLS. Fix: kubectl -n $NS set env deploy/$GATEWAY_DEPLOY COOKIE_SECURE=true"
else
pass "COOKIE_SECURE=true"
fi
if env_is_set ALLOW_INSECURE_COOKIES; then
warn "ALLOW_INSECURE_COOKIES is still set on deploy/$GATEWAY_DEPLOY. It is ignored while COOKIE_SECURE=true but is a footgun: remove it with 'kubectl -n $NS set env deploy/$GATEWAY_DEPLOY ALLOW_INSECURE_COOKIES-'."
fi
echo
echo "== SSO objects and placeholders"
if kubectl -n "$NS" get deployment "$OAUTH_DEPLOY" >/dev/null 2>&1; then
pass "deploy/$OAUTH_DEPLOY exists"
OAUTH_ARGS=$(kubectl -n "$NS" get deployment "$OAUTH_DEPLOY" \
-o jsonpath='{range .spec.template.spec.containers[*].args[*]}{@}{"\n"}{end}')
else
OAUTH_ARGS=""
fail "deploy/$OAUTH_DEPLOY not found. Apply the variant: kubectl apply -f manifests/exposure/gateway-api-sso/"
fi
# GitHub org restriction is the only authorization in the shipped variant.
# --email-domain=* is acceptable only because --github-org narrows membership;
# a missing or empty org value would accept any GitHub account.
if [ -n "$OAUTH_ARGS" ]; then
if printf '%s\n' "$OAUTH_ARGS" | grep -q '^--provider=github$'; then
pass "oauth2-proxy uses --provider=github"
else
fail "oauth2-proxy is not configured with --provider=github. Fix: add --provider=github to deploy/$OAUTH_DEPLOY."
fi
if printf '%s\n' "$OAUTH_ARGS" | grep -Eq '^--github-org=.+$'; then
pass "oauth2-proxy restricts membership with --github-org"
else
fail "oauth2-proxy has no non-empty --github-org. Fix: set --github-org=<your-org> in deploy/$OAUTH_DEPLOY; otherwise --email-domain=* accepts any GitHub account."
fi
fi
if kubectl -n "$NS" get httproute dsh-gateway >/dev/null 2>&1; then
pass "HTTPRoute dsh-gateway exists"
ROUTE_STATE=$(kubectl -n "$NS" get httproute dsh-gateway \
-o jsonpath='{.spec.hostnames[*]}' 2>/dev/null || true)
else
ROUTE_STATE=""
fail "HTTPRoute dsh-gateway not found. Apply the variant: kubectl apply -f manifests/exposure/gateway-api-sso/"
fi
if kubectl -n "$NS" get gateway dsh-gateway >/dev/null 2>&1; then
pass "Gateway dsh-gateway exists"
GATEWAY_STATE=$(kubectl -n "$NS" get gateway dsh-gateway \
-o jsonpath='{range .spec.listeners[*]}{.hostname}{"\n"}{end}' 2>/dev/null || true)
CERT_REFS=$(kubectl -n "$NS" get gateway dsh-gateway \
-o jsonpath='{range .spec.listeners[*].tls.certificateRefs[*]}{.namespace}{"/"}{.name}{"\n"}{end}' 2>/dev/null || true)
else
GATEWAY_STATE=""
CERT_REFS=""
fail "Gateway dsh-gateway not found. Apply the variant: kubectl apply -f manifests/exposure/gateway-api-sso/"
fi
# Placeholders are the default values in the committed manifests. They reach
# the cluster if the variant is applied before editing them.
PLACEHOLDER_RE='your-github-org|dsh-demo\.example\.com'
PLACEHOLDER_STATE=$(printf '%s\n%s\n%s\n' "$OAUTH_ARGS" "$ROUTE_STATE" "$GATEWAY_STATE")
if printf '%s\n' "$PLACEHOLDER_STATE" | grep -En "$PLACEHOLDER_RE" >/dev/null 2>&1; then
fail "the applied SSO objects still contain a placeholder (your-github-org or dsh-demo.example.com):"
printf '%s\n' "$PLACEHOLDER_STATE" | grep -En "$PLACEHOLDER_RE" | sed 's/^/ /' >&2
echo " Replace --github-org, --redirect-url, the Gateway listener host and the HTTPRoute hostname with the real public FQDN and organization, then re-apply and restart the proxy." >&2
else
pass "no your-github-org / dsh-demo.example.com placeholder in the applied SSO objects"
fi
# The Gateway must reference an existing TLS Secret; the default name is
# dsh-demo-tls, but 70-gateway.yaml can be edited to another name.
if [ -n "$CERT_REFS" ]; then
while IFS= read -r ref; do
[ -n "$ref" ] || continue
ref_ns=${ref%%/*}
ref_name=${ref#*/}
ref_ns=${ref_ns:-$NS}
if kubectl -n "$ref_ns" get secret "$ref_name" >/dev/null 2>&1; then
pass "Gateway TLS certificate reference ${ref_ns}/${ref_name} exists"
else
fail "Gateway references TLS Secret ${ref_ns}/${ref_name} but it does not exist. Create it (see manifests/exposure/gateway-api-sso/74-tls-secret.yaml.example)."
fi
done <<<"$CERT_REFS"
else
warn "the Gateway has no TLS certificateRefs; SSO over plain HTTP would expose the session cookie. Terminate TLS at the Gateway listener."
fi
# A second path to the gateway defeats SSO: anyone who can reach
# dsh-gateway:8080 can spoof X-Forwarded-User. This is a best-effort check.
if kubectl -n "$NS" get ingress >/dev/null 2>&1; then
INGRESS_STATE=$(kubectl -n "$NS" get ingress -o json 2>/dev/null || true)
if printf '%s' "$INGRESS_STATE" | grep -q '"name":"dsh-gateway"'; then
fail "an Ingress in namespace $NS also routes to dsh-gateway. With SSO enabled the gateway must be reachable only through oauth2-proxy: delete the Ingress variant (kubectl delete -f manifests/exposure/ingress/)."
fi
fi
echo
if [ "$FAILED" -ne 0 ]; then
echo "preflight-sso: FAILED - fix the items above before exposing the gateway." >&2
exit 1
fi
echo "preflight-sso: ok - the SSO variant looks correctly configured."

38
scripts/check-versions.sh
View File

@@ -22,14 +22,48 @@ expect() { # <file> <literal>
# Main Dockerfile build args
expect Dockerfile "ARG DSH_VERSION=$DSH_VERSION"
expect Dockerfile "ARG DSH_COMMIT=$DSH_COMMIT"
expect Dockerfile "ARG PNPM_VERSION=$PNPM_VERSION"
expect Dockerfile "ARG RELEASE_REVISION=$RELEASE_REVISION"
expect Dockerfile "ARG NODE_BUILD_IMAGE=$NODE_BUILD_IMAGE"
expect Dockerfile "ARG NODE_RUNTIME_IMAGE=$NODE_RUNTIME_IMAGE"
# The build stage must fail closed when the tag no longer resolves to DSH_COMMIT.
expect Dockerfile "git -C /src rev-parse HEAD"
# Example gateway build base image
expect examples/dsh-agents-as-a-service/gateway/Dockerfile \
"ARG GATEWAY_PYTHON_IMAGE=$GATEWAY_PYTHON_IMAGE"
# Local configuration mirrored for humans
expect .env.sample "DSH_VERSION=$DSH_VERSION"
expect .env.sample "DSH_COMMIT=$DSH_COMMIT"
expect .env.sample "PNPM_VERSION=$PNPM_VERSION"
expect .env.sample "RELEASE_REVISION=$RELEASE_REVISION"
# Human-facing docs embed the resolved commit in the build example and table.
expect docs/main-image.md "$DSH_COMMIT"
expect docs/versioning.md "$DSH_COMMIT"
# Every container image is pinned to an immutable multi-arch index digest, not
# a mutable tag. This is a supply-chain invariant: don't "fix" a failing check
# by dropping the digest.
for image_var in NODE_BUILD_IMAGE NODE_RUNTIME_IMAGE GATEWAY_PYTHON_IMAGE OAUTH2_PROXY_IMAGE; do
image_value="${!image_var:-}"
if ! printf '%s' "$image_value" | grep -qE '@sha256:[0-9a-f]{64}$'; then
err "versions.env: $image_var must be digest-pinned (name:tag@sha256:<64 hex>), got: ${image_value:-<unset>}"
fi
done
if ! printf '%s' "${DSH_COMMIT:-}" | grep -qE '^[0-9a-f]{40}$'; then
err "versions.env: DSH_COMMIT must be a full 40-character commit SHA, got: ${DSH_COMMIT:-<unset>}"
fi
# Build entrypoints must pass DSH_COMMIT rather than trusting the tag alone.
expect Makefile "--build-arg DSH_COMMIT=\$(DSH_COMMIT)"
for wf in build-main.yaml release.yaml ci-image.yaml; do
expect ".github/workflows/$wf" "DSH_COMMIT="
done
# Image references in the example manifests
for f in \
examples/dsh-agents-as-a-service/manifests/10-sandboxtemplate.yaml \
@@ -38,6 +72,10 @@ for f in \
done
expect examples/dsh-agents-as-a-service/manifests/50-gateway.yaml "image: $GATEWAY_IMAGE:$IDENTITY"
# The oauth2-proxy exposure variant pins the digest, not the mutable tag.
expect examples/dsh-agents-as-a-service/manifests/exposure/gateway-api-sso/72-oauth2-proxy.yaml \
"image: $OAUTH2_PROXY_IMAGE"
# The gateway image must live below the repository image (one package per
# example component, no unrelated top-level packages).
case "$GATEWAY_IMAGE" in

51
scripts/set-version.sh
View File

@@ -4,6 +4,11 @@
#
# ./scripts/set-version.sh dsh-v0.2.2-rc.1 # new DSH ref, no revision
# ./scripts/set-version.sh dsh-v0.2.1-rc.5 r3 # same ref, new repo revision
#
# For a new DSH ref the script resolves the tag's commit with "git ls-remote"
# (peeled ^{} commit first, tag ref as fallback) and rewrites DSH_COMMIT in
# versions.env, the Dockerfile ARG default and .env.sample. A revision-only bump
# at the same ref keeps the existing commit and needs no network access.
set -euo pipefail
cd "$(dirname "$0")/.."
@@ -23,9 +28,11 @@ if [ -n "$new_revision" ]; then
fi
# shellcheck source=/dev/null
old_version="$(. ./versions.env; printf '%s' "$DSH_VERSION")"
# shellcheck source=/dev/null
old_revision="$(. ./versions.env; printf '%s' "${RELEASE_REVISION:-}")"
. ./versions.env
old_version="$DSH_VERSION"
old_revision="${RELEASE_REVISION:-}"
old_commit="${DSH_COMMIT:-}"
old_identity="$old_version"
[ -z "$old_revision" ] || old_identity="$old_version-$old_revision"
@@ -37,6 +44,29 @@ if [ "$old_identity" = "$new_identity" ]; then
exit 0
fi
# Resolve the commit the new upstream tag points at. Annotated tags expose a
# peeled ^{} line (the commit) and a tag-object line; prefer the peeled commit
# and fall back to the tag ref for lightweight tags. A repository-only revision
# bump keeps the already pinned commit, so it needs no network.
new_commit="$old_commit"
if [ "$new_version" != "$old_version" ] || [ -z "$new_commit" ]; then
if ! refs="$(git ls-remote --tags --exit-code "$DSH_REPO" \
"refs/tags/$new_version^{}" "refs/tags/$new_version")"; then
echo "set-version: upstream tag '$new_version' not found in $DSH_REPO" >&2
exit 1
fi
new_commit="$(printf '%s\n' "$refs" \
| awk -v t="refs/tags/$new_version^{}" '$2 == t {print $1}' | head -n1)"
if [ -z "$new_commit" ]; then
new_commit="$(printf '%s\n' "$refs" \
| awk -v t="refs/tags/$new_version" '$2 == t {print $1}' | head -n1)"
fi
if ! printf '%s' "$new_commit" | grep -qE '^[0-9a-f]{40}$'; then
echo "set-version: could not resolve a commit for '$new_version' in $DSH_REPO" >&2
exit 1
fi
fi
# Replace the identity first (it contains the base version), then the base
# version itself. CHANGELOG.md is generated from tags, so it is left alone.
while IFS= read -r file; do
@@ -48,17 +78,30 @@ while IFS= read -r file; do
sed -i "s/$old_identity/$new_identity/g; s/$old_version/$new_version/g" "$file"
done < <(git ls-files)
# versions.env is authoritative: write both keys explicitly so an empty
# versions.env is authoritative: write these keys explicitly so an empty
# revision is stored as an empty value.
sed -i "s/^DSH_VERSION=.*/DSH_VERSION=$new_version/" versions.env
sed -i "s/^RELEASE_REVISION=.*/RELEASE_REVISION=$new_revision/" versions.env
sed -i "s/^DSH_COMMIT=.*/DSH_COMMIT=$new_commit/" versions.env
# Values stored separately from the full identity: Dockerfile ARG defaults and
# the human-facing sample cannot be reached by the identity replacement above.
sed -i "s/^ARG DSH_VERSION=.*/ARG DSH_VERSION=$new_version/" Dockerfile
sed -i "s/^ARG RELEASE_REVISION=.*/ARG RELEASE_REVISION=$new_revision/" Dockerfile
sed -i "s/^ARG DSH_COMMIT=.*/ARG DSH_COMMIT=$new_commit/" Dockerfile
sed -i "s/^DSH_VERSION=.*/DSH_VERSION=$new_version/" .env.sample
sed -i "s/^RELEASE_REVISION=.*/RELEASE_REVISION=$new_revision/" .env.sample
sed -i "s/^DSH_COMMIT=.*/DSH_COMMIT=$new_commit/" .env.sample
# The human-facing build examples embed the full SHA, which the version
# replacement above cannot see. Keep them in step; check-versions.sh verifies.
if [ -n "$old_commit" ] && [ "$old_commit" != "$new_commit" ]; then
for doc in docs/main-image.md docs/versioning.md; do
[ -f "$doc" ] || continue
sed -i "s/$old_commit/$new_commit/g" "$doc"
done
fi
echo "set-version: $old_identity -> $new_identity"
echo "set-version: DSH_COMMIT=$new_commit"
echo "set-version: run 'make check' and review the diff"

16
versions.env
View File

@@ -2,10 +2,17 @@
# repository. Consumed by the Makefile, build.sh, scripts/*.sh and the GitHub
# Actions workflows. Never hard-code these values elsewhere: run "make check"
# (scripts/check-versions.sh) to verify every pin.
#
# Every container image below is pinned to an immutable multi-arch index digest
# (name:tag@sha256:...), not just a mutable tag. Bump the tag and the digest
# together; scripts/check-versions.sh requires the digest form.
# Upstream DeepSeek Harness source, and the ref this repository packages.
# DSH_COMMIT is the full commit SHA that DSH_VERSION resolves to upstream; the
# Dockerfile verifies the clone's HEAD against it and fails the build on drift.
DSH_REPO=https://github.com/deepseek-ai/deepseek-harness.git
DSH_VERSION=dsh-v0.2.1-rc.5
DSH_COMMIT=b830ec950faba0a46dda564be3e2d854afed94f1
PNPM_VERSION=11.28.5
# Release identity = DSH_VERSION plus an optional repository revision.
@@ -15,6 +22,13 @@ PNPM_VERSION=11.28.5
# repository-only change at the same ref becomes -r2.
RELEASE_REVISION=
# Base images for the main DSH Web image (root Dockerfile), pinned by digest.
NODE_BUILD_IMAGE=docker.io/node:24-trixie@sha256:8c761eca2bf7eec965f946cf47b8090143cb41f45f2f8699af0cc0298d2d2d19
NODE_RUNTIME_IMAGE=docker.io/node:24-trixie-slim@sha256:54f84b4cf8432b21aaffd5d3ce8d41e12cea0d10d33c09d2f3c9879d49d6b446
# Base image for the example gateway (examples/dsh-agents-as-a-service/gateway).
GATEWAY_PYTHON_IMAGE=docker.io/python:3.12-slim@sha256:aa943ce411423c7582288f18707ff66eddcbaeb510502d747a72aeb34f30d159
# Control-plane versions used by the examples and their manifests.
# The matching SHA-256s pin the release manifests that CI downloads and
# applies; update both the version and the digest when bumping either pin.
@@ -22,7 +36,7 @@ AGENT_SANDBOX_VERSION=v1.0.9
AGENT_SANDBOX_MANIFEST_SHA256=4bf9ee5558e9a663f91cfa5963e96f2591bfaa1dcf79f03bfab65bc5d65cc699
GATEWAY_API_VERSION=v1.2.1
GATEWAY_API_MANIFEST_SHA256=97598bf6ab3b33b9b5c5432bdd24de091e4e9c3aa0575ebb0710a2a19cd64d64
OAUTH2_PROXY_IMAGE=quay.io/oauth2-proxy/oauth2-proxy:v7.15.5
OAUTH2_PROXY_IMAGE=quay.io/oauth2-proxy/oauth2-proxy:v7.15.5@sha256:8498b0d0ef0a7b29686414000a08aee467f02d0299c9ed1e006a8f33fc017916
# Registry layout: the main image is the repository itself, and every example
# component is a nested package below it (<registry>/<owner>/<repo>/<example>/<component>).