diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 0000000..435bd5d --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,65 @@ +name: Validate + +# Validates the Kubernetes manifests against the real agent-sandbox CRD +# schema (server-side dry-run on a throwaway kind cluster) and runs the +# gateway unit tests. Runs on every PR and on pushes to main. +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + validate-manifests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + with: + version: v0.23.0 + cluster_name: dsh-validate + + - name: Install agent-sandbox (extensions) + run: | + VERSION=v1.0.7 + kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${VERSION}/sandbox-with-extensions.yaml + kubectl wait --for=condition=Established crd/sandboxtemplates.extensions.agents.x-k8s.io --timeout=180s + kubectl wait --for=condition=Established crd/sandboxwarmpools.extensions.agents.x-k8s.io --timeout=180s + kubectl wait --for=condition=Established crd/sandboxclaims.extensions.agents.x-k8s.io --timeout=180s + kubectl wait --for=condition=Established crd/sandboxes.agents.x-k8s.io --timeout=180s + + - name: Validate manifests (server dry-run) + run: | + set -euo pipefail + kubectl create namespace dsh-demo + for f in examples/dsh-agents-as-a-service/*.yaml; do + echo "== $f ==" + kubectl apply --dry-run=server -f "$f" + done + kubectl apply --dry-run=server -f deploy/k8s/sandbox-template.yaml + echo "all manifests validated" + + gateway-tests: + runs-on: ubuntu-latest + defaults: + run: + working-directory: examples/dsh-agents-as-a-service/gateway + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r requirements.txt pytest + + - name: Run gateway unit tests + run: python -m pytest test_gateway.py -v diff --git a/.gitignore b/.gitignore index bb6f512..a80a91a 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,7 @@ vite.config.ts.timestamp-* .turbo !.env.sample + +# Python +__pycache__/ +*.py[cod] diff --git a/README.md b/README.md index 970b37d..ce098d7 100644 --- a/README.md +++ b/README.md @@ -1,178 +1,78 @@ -# deepseek-kubernetes-sandbox based on hermes-agents-as-a-service +# deepseek-kubernetes-sandbox -Original: https://github.com/kubernetes-sigs/agent-sandbox/tree/main/examples/hermes-agents-as-a-service +Packages [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) Web (`dsh web`) into +a multi-arch, Kubernetes-ready container image, and ships the +[agents-as-a-service](examples/dsh-agents-as-a-service/) platform pattern — running a personal +DSH Web instance per user on [kubernetes-sigs/agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox). -After analyzing the `hermes-agents-as-a-service` example, its core pattern maps directly onto your `dsh-web` container. The example's key mechanisms are: **`SandboxTemplate` defines a reusable Pod blueprint, `SandboxWarmPool` pre-warms instances, `SandboxClaim` lets each user claim a persistent Sandbox, and `service: true` gives each Sandbox a stable network identity for gateway routing.** Your DSH container is a strong fit for this model — it needs a persistent `DSH_HOME` (like Hermes needs a persistent data directory) and exposes a web port. The main differences are how secrets are injected and how the port is surfaced to end users. +## What's in this repo -### 🔑 Mechanism Mapping: Hermes → DSH +| Path | Purpose | +| --- | --- | +| [`Dockerfile`](Dockerfile) | Multi-stage build that clones DSH, `pnpm pack`s the CLI, and produces a slim runtime image | +| [`build.sh`](build.sh) | Local build + run (builds `dsh-web:` and runs it) | +| [`.github/workflows/multi-build.yaml`](.github/workflows/multi-build.yaml) | CI build (amd64 + arm64) and push to GHCR | +| [`deploy/k8s/sandbox-template.yaml`](deploy/k8s/sandbox-template.yaml) | Minimal single-tenant `SandboxTemplate` | +| [`examples/dsh-agents-as-a-service/`](examples/dsh-agents-as-a-service/) | The full multi-tenant platform pattern (warm pool, per-user claims, gateway, smoke test) | -| Hermes example mechanism | Purpose | Adaptation for DSH | -| :--- | :--- | :--- | -| **`SandboxTemplate`** | Reusable Pod blueprint | Change image to `dsh-web:0.2.1-rc.2`, expose port `3080` | -| **`volumeClaimTemplates`** | Auto-create a PVC per user, persisting state | Mount point becomes `/home/node/.dsh` (DSH's state directory) | -| **`SandboxWarmPool`** | Pre-warmed instances for sub-second provisioning | Keep as-is; size the pool for expected concurrent users | -| **`SandboxClaim` + `warmPoolRef`** | User "registration" claims a lightweight object, adopts a pooled instance | Keep exactly as-is; one Claim per DSH user | -| **`service: true`** | Creates a headless Service per Sandbox with a stable DNS name | **Enable it** so a front-end gateway or Ingress can reliably route to each user's DSH Web | -| **`envVarsInjectionPolicy`** | Controls whether a Claim may inject/override env vars | DSH needs `DEEPSEEK_API_KEY` injected, so either allow it or reference a Secret from the template | -| **`operatingMode`** | Cost switch: `Running` / `Suspended` (deletes Pod, keeps PVC + Service) | Reuse directly — suspend idle DSH instances to save cost | +## Quick start (image) -### 📦 Equivalent Kubernetes Manifests - -The manifests below mirror the Hermes example's organization. Apply them in order, all in a dedicated namespace (e.g. `dsh-demo`). - -#### 1. Secret: DSH API key - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: dsh-api-secret - namespace: dsh-demo -type: Opaque -stringData: - DEEPSEEK_API_KEY: "sk-..." # replace with a real key -``` - -#### 2. SandboxTemplate: DSH Web blueprint - -This is the core resource, corresponding to `10-sandboxtemplate.yaml` in the example. Key settings: **`service: true`** for stable DNS, **`volumeClaimTemplates`** to persist `/home/node/.dsh`, and a security context that runs as the non-root `node` user (UID 1000). - -```yaml -apiVersion: extensions.agents.x-k8s.io/v1beta1 -kind: SandboxTemplate -metadata: - name: dsh-web-template - namespace: dsh-demo -spec: - service: true # headless Service per Sandbox → stable DNS - podTemplate: - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - containers: - - name: dsh-web - image: dsh-web:0.2.1-rc.2 # replace with your registry path - ports: - - containerPort: 3080 - name: web - env: - - name: DEEPSEEK_API_KEY - valueFrom: - secretKeyRef: - name: dsh-api-secret - key: DEEPSEEK_API_KEY - - name: DSH_PUBLIC_URL - value: "http://localhost:3080" # change to the external URL when behind a gateway/Ingress - volumeMounts: - - name: dsh-home - mountPath: /home/node/.dsh - readinessProbe: - httpGet: - path: / - port: 3080 - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: - path: / - port: 3080 - initialDelaySeconds: 15 - periodSeconds: 20 - # If your cluster uses gVisor / Kata, set the runtime class here - # runtimeClassName: gvisor - volumeClaimTemplates: - - metadata: - name: dsh-home - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 5Gi # adjust to your DSH state size -``` - -#### 3. SandboxWarmPool: pre-warmed pool - -Corresponds to `20-sandboxwarmpool.yaml`. Pooled Sandboxes pull the image and boot ahead of time; a user Claim adopts one instantly. - -```yaml -apiVersion: extensions.agents.x-k8s.io/v1beta1 -kind: SandboxWarmPool -metadata: - name: dsh-web-pool - namespace: dsh-demo -spec: - replicas: 2 # number of pre-warmed instances - sandboxTemplateRef: - name: dsh-web-template -``` - -#### 4. SandboxClaim: per-user claim - -Corresponds to `40-claim.yaml`. Each user creates one lightweight Claim and receives a pooled instance with its own PVC and stable DNS. - -```yaml -apiVersion: extensions.agents.x-k8s.io/v1beta1 -kind: SandboxClaim -metadata: - name: user-alice-dsh - namespace: dsh-demo -spec: - sandboxTemplateRef: - name: dsh-web-template - warmPoolRef: - name: dsh-web-pool - # If the template allows env injection (envVarsInjectionPolicy: Allowed), - # you can override/add variables here: - # env: - # - name: DSH_PUBLIC_URL - # value: "https://alice.dsh.example.com" -``` - -### 🌐 Networking: How Users Reach DSH Web - -Unlike Hermes agents, which are mainly consumed by an internal gateway, DSH Web is a **user-facing web UI** — each Sandbox's `3080` port must be reachable by end users. Two patterns: - -#### Pattern A: Direct via headless Service (simple) - -With `service: true`, each Sandbox gets a headless Service whose DNS name is exposed in the Sandbox's `status.serviceFQDN`. You can reach it from inside the cluster via that FQDN, or add an Ingress/upper-layer gateway that routes a path to the corresponding Service. - -#### Pattern B: Gateway-proxied (recommended, mirrors Hermes's service layer) - -The Hermes example ships a ~250-line Python gateway (`gateway/gateway.py`) that handles **per-user routing, auth, and auto-waking suspended Sandboxes**. You can build a simpler equivalent for DSH: - -1. Accept external requests (e.g. `/u/{user}/**`). -2. Look up the `status.serviceFQDN` of that user's Sandbox. -3. Reverse-proxy to `http://{serviceFQDN}:3080`. -4. If the Sandbox is `Suspended`, flip its `operatingMode` back to `Running`, wait for `Ready`, then proxy. - -If you don't want a gateway yet, you can manually create one Ingress per user pointing at their Service — less automated, but functional. - -### ⚙️ Key Differences and Caveats - -1. **Secret-injection policy.** The Hermes example sets `envVarsInjectionPolicy: Disallowed` to strictly limit what Claims can inject. DSH needs `DEEPSEEK_API_KEY` injected, so you have two choices: - - **Set it to `Allowed`**, then reference the Secret from `SandboxClaim.env` (more flexible, but requires trusting Claim creators). - - **Keep it `Disallowed`** and reference the Secret directly inside the `SandboxTemplate`'s `podTemplate` (stricter, but all users share the same key — fine for a single-tenant internal setup). - Pick based on your multi-tenancy model. - -2. **`DSH_PUBLIC_URL` configuration.** Your entrypoint defaults to `http://localhost:3080`. When accessed via a headless Service or gateway, set `DSH_PUBLIC_URL` to the real external URL (e.g. `https://alice.dsh.example.com`), otherwise links DSH generates may be unreachable. You can override it in `SandboxClaim.env` (if injection is allowed) or rewrite URLs in the gateway. - -3. **Image pull and platform.** As with the Hermes example, push `dsh-web` to a registry reachable by the cluster. Your image is built for `amd64`; node architecture must match. - -4. **PVC size.** DSH state size (sessions, configs, etc.) depends on usage; Hermes allocated a PVC for `/opt/data`. Adjust the `storage` request in `volumeClaimTemplates` accordingly. - -5. **Auto-suspend via `operatingMode`.** Hermes's gateway includes an idle sweeper that suspends idle instances automatically. If your DSH gateway implements a similar sweeper, you get the same cost model: `Running` while active, `Suspended` when idle (PVC + Service retained), auto-woken when the user returns. - -### ✅ Deployment Order +The image is published to GitHub Container Registry as +`ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:`, where `` is `main`, +`latest`, or a DSH version tag such as `dsh-v0.2.1-rc.2`. ```bash -kubectl create namespace dsh-demo -kubectl apply -f 00-secret.yaml -kubectl apply -f 10-sandboxtemplate.yaml -kubectl apply -f 20-sandboxwarmpool.yaml -kubectl apply -f 40-claim.yaml # one per user +docker run --rm -it \ + -p 3080:3080 \ + -v dsh-home:/home/node/.dsh \ + -e DEEPSEEK_API_KEY \ + ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.2 ``` -After applying, watch the warm pool spin up with `kubectl -n dsh-demo get sandboxes -w`, and inspect adoption and `status.serviceFQDN` with `kubectl -n dsh-demo get sandboxclaim user-alice-dsh -o yaml`. At that point you have a platform structurally identical to the Hermes example, but running DSH Web for multiple users. +> **Local port note:** `dsh web` listens on `3080` inside the container. If `3080` is +> already in use on your workstation, map a different host port (e.g. `-p 18081:3080`). + +## The multi-user platform pattern + +`examples/dsh-agents-as-a-service/` is a port of the upstream +[hermes-agents-as-a-service](https://github.com/kubernetes-sigs/agent-sandbox/tree/main/examples/hermes-agents-as-a-service) +example to DSH Web. It demonstrates the platform shape — one `SandboxTemplate`, one warm pool, +one tiny `SandboxClaim` per user, and `operatingMode` as the cost dial — plus a gateway that +turns it into a usable service (signup, per-user tokens, `/u/{user}/**` proxy with +wake-on-connect, idle suspend). See its [README](examples/dsh-agents-as-a-service/README.md) for +the full walkthrough. + +### Mechanism mapping: Hermes → DSH + +| Hermes example mechanism | Role | Adaptation for DSH | +| :--- | :--- | :--- | +| `SandboxTemplate` | Reusable Pod blueprint | Image `dsh-web`, port `3080` | +| `volumeClaimTemplates` | Auto-create a PVC per user | Mount `/home/node/.dsh` (DSH state) | +| `SandboxWarmPool` | Pre-warmed instances | Keep as-is; size for expected concurrency | +| `SandboxClaim` + `warmPoolRef` | "Registration" adopts a pooled instance | Keep exactly as-is; one claim per user | +| `service: true` | Stable DNS per Sandbox | Enabled, so the gateway routes to each user's DSH Web | +| `envVarsInjectionPolicy: Disallowed` | Reject claims that would cold-start | `DEEPSEEK_API_KEY` injected via `envFrom` on the template (shared platform key) | +| `operatingMode` | Cost switch: Running / Suspended | Reused directly — suspend idle instances to save cost | + +### Key differences from Hermes + +- **Multi-arch** — the DSH image is built for amd64 and arm64 (no amd64 nodeSelector). +- **Single port** — DSH Web serves everything on `3080`; no API/dashboard port split. +- **One secret** — only `DEEPSEEK_API_KEY`; no dashboard/session/api-server credentials. +- **Host forwarding** — the gateway forwards `Host` + `X-Forwarded-*` (and keeps + `DSH_PUBLIC_URL` empty) so DSH generates reachable links. +- **WebSocket caveat** — the requests-based proxy handles SSE but not WebSocket upgrades; use a + path-based Ingress per user for full web-GUI interactivity. + +## Build it yourself + +```bash +./build.sh # docker build + run (dsh-web:0.2.1-rc.2) +# or +docker build \ + --build-arg DSH_VERSION=dsh-v0.2.1-rc.2 \ + -t dsh-web:0.2.1-rc.2 . +``` + +See [`Dockerfile`](Dockerfile) for build args (`DSH_REPO`, `DSH_VERSION`, `PNPM_VERSION`) and +the runtime image contract (non-root `node` user, `DSH_HOME=/home/node/.dsh`, `EXPOSE 3080`). diff --git a/deploy/k8s/sandbox-template.yaml b/deploy/k8s/sandbox-template.yaml index 230f304..b45a1c4 100644 --- a/deploy/k8s/sandbox-template.yaml +++ b/deploy/k8s/sandbox-template.yaml @@ -1,30 +1,53 @@ -apiVersion: extensions.agents.x-k8s.io/v1alpha1 +# Minimal single-tenant SandboxTemplate for DeepSeek Harness Web. +# +# This is the "just one DSH Web" quick-start template. For the full +# multi-tenant platform (warm pool, per-user claims, gateway, idle-suspend), +# see examples/dsh-agents-as-a-service/. +apiVersion: extensions.agents.x-k8s.io/v1beta1 kind: SandboxTemplate metadata: name: dsh-web-template - namespace: default + namespace: dsh-demo spec: + service: true # headless Service per Sandbox -> stable DNS (status.serviceFQDN) podTemplate: spec: securityContext: runAsNonRoot: true runAsUser: 1000 # Matches the UID of the 'node' user runAsGroup: 1000 + fsGroup: 1000 containers: - name: dsh-web - image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox/deepseek-kubernetes-sandbox:0.2.1-rc.2 + image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.2 + imagePullPolicy: IfNotPresent ports: - - containerPort: 3080 - env: - - name: DEEPSEEK_API_KEY - valueFrom: - secretKeyRef: - name: deepseek-secret - key: api-key + - name: web + containerPort: 3080 + envFrom: + - secretRef: + name: dsh-provider-keys # Secret with key DEEPSEEK_API_KEY volumeMounts: - name: dsh-home - mountPath: /home/node/.dsh - volumes: - - name: dsh-home - persistentVolumeClaim: - claimName: dsh-home-pvc # You will need to create this PVC \ No newline at end of file + mountPath: /home/node/.dsh # DSH_HOME + readinessProbe: + httpGet: + path: / + port: 3080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: / + port: 3080 + initialDelaySeconds: 15 + periodSeconds: 15 + volumeClaimTemplates: + - metadata: + name: dsh-home + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + # storageClassName omitted => cluster default diff --git a/examples/dsh-agents-as-a-service/00-prereqs.yaml b/examples/dsh-agents-as-a-service/00-prereqs.yaml new file mode 100644 index 0000000..7d82f77 --- /dev/null +++ b/examples/dsh-agents-as-a-service/00-prereqs.yaml @@ -0,0 +1,27 @@ +# DSH "Agents as a Service" — prerequisites: namespace + shared provider key. +# +# The dsh-provider-keys Secret is deliberately NOT committed with a usable +# value. DEEPSEEK_API_KEY reaches every tenant sandbox (injected via envFrom +# on the SandboxTemplate, stamped before any user exists — a warm-pool +# constraint), so a real key committed here would be shared by every install +# of this example. Generate a fresh value per install (README step 1; +# run-test-kind.sh uses a throwaway): +# +# kubectl -n dsh-demo create secret generic dsh-provider-keys \ +# --from-literal=DEEPSEEK_API_KEY="sk-..." +# +# All the infrastructure in this example works without a real key; only +# actual model calls need one. +apiVersion: v1 +kind: Namespace +metadata: + name: dsh-demo +--- +apiVersion: v1 +kind: Secret +metadata: + name: dsh-provider-keys + namespace: dsh-demo +type: Opaque +# stringData: +# DEEPSEEK_API_KEY: "sk-your-key-here" diff --git a/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml new file mode 100644 index 0000000..dcd3fe0 --- /dev/null +++ b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml @@ -0,0 +1,81 @@ +# The reusable DSH Web blueprint. Warm pools stamp pod env BEFORE any user +# exists, so per-user secrets cannot live in pod env; the shared provider key +# is injected via envFrom, and claims that would bypass the warm pool are +# rejected outright (see 40-claim-rejected.yaml). +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxTemplate +metadata: + name: dsh-web + namespace: dsh-demo +spec: + # Claims must not inject env or volumeClaimTemplates: with Disallowed, such + # claims are REJECTED instead of silently taking the cold (non-warm-pool) + # provisioning path. Demonstrated in step 6. + envVarsInjectionPolicy: Disallowed + volumeClaimTemplatesPolicy: Disallowed + + # Per-sandbox headless Service -> stable DNS (sandbox.status.serviceFQDN). + # The gateway routes each user to their DSH Web instance by this name. + service: true + + # No gateway pod scopes ingress in this standalone demo, so the + # NetworkPolicy is left unmanaged. A production platform would use + # networkPolicyManagement: Managed and restrict ingress to its gateway on + # :3080 — with the shared provider key, that NetworkPolicy is the per-user + # isolation boundary. + networkPolicyManagement: Unmanaged + + podTemplate: + spec: + # DSH Web is multi-arch (amd64 + arm64); no arch nodeSelector needed. + restartPolicy: "OnFailure" + securityContext: + runAsUser: 1000 # the 'node' user in the DSH image + runAsGroup: 1000 + fsGroup: 1000 + runAsNonRoot: true + containers: + - name: dsh-web + image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.2 + imagePullPolicy: IfNotPresent + ports: + - name: web + containerPort: 3080 + envFrom: + - secretRef: + name: dsh-provider-keys + env: + # Empty => DSH derives links from the request Host, which the + # gateway forwards (plus X-Forwarded-*). Point this at the + # gateway base URL if you want hard-coded absolute links instead. + - name: DSH_PUBLIC_URL + value: "" + readinessProbe: + httpGet: + path: / + port: 3080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 30 + livenessProbe: + httpGet: + path: / + port: 3080 + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + resources: + requests: {cpu: 250m, memory: 512Mi} + limits: {cpu: "1", memory: 1Gi} + volumeMounts: + - name: dsh-home + mountPath: /home/node/.dsh # DSH_HOME: sessions, config, state + volumeClaimTemplates: + - metadata: + name: dsh-home + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + # storageClassName omitted => cluster default (kind & GKE both work) diff --git a/examples/dsh-agents-as-a-service/20-sandboxwarmpool.yaml b/examples/dsh-agents-as-a-service/20-sandboxwarmpool.yaml new file mode 100644 index 0000000..c3bc768 --- /dev/null +++ b/examples/dsh-agents-as-a-service/20-sandboxwarmpool.yaml @@ -0,0 +1,12 @@ +# Pre-warmed spares: image pulled, PVC provisioned, DSH Web booted, probe +# green — everything a signup would otherwise wait for is pre-paid, so +# claiming an instance takes ~2s instead of a full cold provision. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxWarmPool +metadata: + name: dsh-pool + namespace: dsh-demo +spec: + replicas: 2 + sandboxTemplateRef: + name: dsh-web diff --git a/examples/dsh-agents-as-a-service/30-claim-alice.yaml b/examples/dsh-agents-as-a-service/30-claim-alice.yaml new file mode 100644 index 0000000..beaad28 --- /dev/null +++ b/examples/dsh-agents-as-a-service/30-claim-alice.yaml @@ -0,0 +1,19 @@ +# "User signup": one claim per user. This is the entire per-user resource — +# the platform pattern keeps it deliberately minimal: +# +# - warmPoolRef ONLY. No spec.env and no spec.volumeClaimTemplates: either +# would force a cold start past the warm pool (and this template's +# Disallowed policies reject them outright — see 40-claim-rejected.yaml). +# - additionalPodMetadata is the one warm-compatible per-claim +# customization. Label keys must use the sandbox.users.io domain. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxClaim +metadata: + name: dsh-alice + namespace: dsh-demo +spec: + warmPoolRef: + name: dsh-pool + additionalPodMetadata: + labels: + sandbox.users.io/dsh-user: alice diff --git a/examples/dsh-agents-as-a-service/40-claim-rejected.yaml b/examples/dsh-agents-as-a-service/40-claim-rejected.yaml new file mode 100644 index 0000000..2849c5d --- /dev/null +++ b/examples/dsh-agents-as-a-service/40-claim-rejected.yaml @@ -0,0 +1,18 @@ +# NEGATIVE demo: this claim carries spec.env, which the template's +# envVarsInjectionPolicy: Disallowed rejects. The claim never binds a +# sandbox — inspect it to see the rejection (step 6 of the README). +# +# Why a platform wants rejection over acceptance: env injection forces a +# cold start that bypasses the warm pool, silently turning a ~2s signup into +# a cold provision. Disallowed turns that mistake into a loud error. +apiVersion: extensions.agents.x-k8s.io/v1beta1 +kind: SandboxClaim +metadata: + name: dsh-mallory + namespace: dsh-demo +spec: + warmPoolRef: + name: dsh-pool + env: + - name: SNEAKY_OVERRIDE + value: "should-not-work" diff --git a/examples/dsh-agents-as-a-service/50-gateway.yaml b/examples/dsh-agents-as-a-service/50-gateway.yaml new file mode 100644 index 0000000..8ca6bb5 --- /dev/null +++ b/examples/dsh-agents-as-a-service/50-gateway.yaml @@ -0,0 +1,94 @@ +# The gateway: control+data plane that turns the resources into a usable +# service. Build the image first (see README step 8): +# docker build -t dsh-aaas-gateway:demo gateway/ +# kind load docker-image dsh-aaas-gateway:demo # (kind clusters) +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dsh-gateway + namespace: dsh-demo +--- +# Least privilege: claims are created/deleted per user; sandboxes are only +# read and patched (operatingMode flips for suspend/resume). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: dsh-gateway + namespace: dsh-demo +rules: + - apiGroups: ["extensions.agents.x-k8s.io"] + resources: ["sandboxclaims"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes"] + verbs: ["get", "list", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dsh-gateway + namespace: dsh-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: dsh-gateway +subjects: + - kind: ServiceAccount + name: dsh-gateway + namespace: dsh-demo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dsh-gateway + namespace: dsh-demo +spec: + replicas: 1 # in-memory idle clock => single replica by design + selector: + matchLabels: + app: dsh-gateway + template: + metadata: + labels: + app: dsh-gateway + spec: + serviceAccountName: dsh-gateway + securityContext: + runAsNonRoot: true + runAsUser: 10000 # matches USER in gateway/Dockerfile + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + containers: + - name: gateway + image: dsh-aaas-gateway:demo + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + ports: + - containerPort: 8080 + env: + - name: NAMESPACE + value: dsh-demo + - name: POOL + value: dsh-pool + - name: IDLE_TIMEOUT + value: "60" + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: 500m, memory: 256Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: dsh-gateway + namespace: dsh-demo +spec: + selector: + app: dsh-gateway + ports: + - port: 8080 + targetPort: 8080 diff --git a/examples/dsh-agents-as-a-service/README.md b/examples/dsh-agents-as-a-service/README.md new file mode 100644 index 0000000..21c0072 --- /dev/null +++ b/examples/dsh-agents-as-a-service/README.md @@ -0,0 +1,198 @@ +# DeepSeek Harness Web as a Service: the multi-user platform pattern + +This example distills the upstream +[hermes-agents-as-a-service](https://github.com/kubernetes-sigs/agent-sandbox/tree/main/examples/hermes-agents-as-a-service) +platform pattern into pure agent-sandbox resources + kubectl, running **DeepSeek Harness Web +(DSH)** instead of Hermes. Every user of such a platform gets a personal, long-lived DSH Web +instance that is **provisioned in ~2 seconds from a warm pool, suspended when idle to save +cost, and woken transparently when the user returns** — with all state (conversations, +sessions, config) surviving on a per-user PVC at `/home/node/.dsh`. + +It ships in two layers: + +1. **The resource layer** (steps 1–7): pure manifests + kubectl, so every mechanism is visible. +2. **The service layer** (step 8): a small included **gateway** (`gateway/`) that makes it + directly usable — signup API, per-user bearer tokens, a `/u/{user}/**` proxy that + transparently **wakes suspended instances on connect**, and an idle sweeper that suspends + them again. + +What it demonstrates, feature by feature: + +| agent-sandbox feature | Role in the platform | +| --- | --- | +| `SandboxTemplate` with `volumeClaimTemplates` | Identical DSH pods; per-user state on a PVC at `/home/node/.dsh` | +| `envVarsInjectionPolicy` / `volumeClaimTemplatesPolicy: Disallowed` | Claims that would bypass the warm pool are **rejected loudly**, not cold-started silently | +| `service: true` | Stable per-sandbox DNS (`status.serviceFQDN`) for the gateway to route on | +| `SandboxWarmPool` | Signup latency is pre-paid: image pull, PVC provision, DSH boot all happen before any user exists | +| `SandboxClaim` + `warmPoolRef` | "Signup" = create a ~10-line claim; adoption is a label flip, Ready in ~2s | +| `additionalPodMetadata` | The one warm-compatible per-claim customization (user label on the pod) | +| `Sandbox.spec.operatingMode` | Suspend = delete only the pod (PVC + Service survive); resume = recreate pod, reattach the same PVC | +| Cascade delete | Deleting the claim garbage-collects sandbox **and** PVC — account deletion in one call | + +## Differences from the Hermes example + +- **Multi-arch**: the DSH image is built for amd64 and arm64, so there is no + `kubernetes.io/arch: amd64` nodeSelector. +- **Single port**: DSH Web serves everything on `3080`; the gateway proxies to one upstream + instead of splitting `/v1/*` (API) from the dashboard. +- **One shared secret**: only `DEEPSEEK_API_KEY` is injected (via `envFrom`); there is no + dashboard basic-auth, session-cookie secret, or API-server key. +- **Host forwarding**: the gateway forwards the original `Host` + `X-Forwarded-*` (and keeps + `DSH_PUBLIC_URL` empty) so DSH generates reachable links from the request. +- **WebSocket caveat**: the requests-based streaming proxy handles SSE but not WebSocket + upgrades. For full DSH web-GUI interactivity, front the gateway with a path-based Ingress + per user (or a WS-capable proxy). + +## Prerequisites + +- A Kubernetes cluster with a default StorageClass (kind and GKE both work as-is). +- agent-sandbox with extensions installed: + + ```bash + VERSION=v1.0.7 + kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${VERSION}/sandbox-with-extensions.yaml + kubectl -n agent-sandbox-system rollout status deployment --timeout=180s + ``` + +Or run everything below in one shot with `./run-test-kind.sh` (expects a kind cluster + the +manifest above already applied). + +## 1. Install the platform resources + +First the namespace, then the shared provider key. The key reaches **every** tenant sandbox +(a warm-pool constraint: pod env is stamped before any user exists), so it is **never +committed** — generate a fresh value per install: + +```bash +kubectl apply -f 00-prereqs.yaml +kubectl -n dsh-demo create secret generic dsh-provider-keys \ + --from-literal=DEEPSEEK_API_KEY="sk-..." +kubectl apply -f 10-sandboxtemplate.yaml -f 20-sandboxwarmpool.yaml +``` + +Watch the two warm spares boot (first time pulls the image — subsequent operations never pay +that again; that is the point of the pool): + +```bash +kubectl -n dsh-demo get sandboxes -l agents.x-k8s.io/warm-pool-sandbox -w +``` + +Wait until both show `READY: True`. Each spare already has its PVC bound and DSH Web serving +on port `3080`. + +## 2. "Signup": claim an instance (~2s) + +A user signing up is one small claim — `warmPoolRef` plus a pod label: + +```bash +kubectl apply -f 30-claim-alice.yaml +kubectl -n dsh-demo wait sandboxclaim dsh-alice --for=condition=Ready --timeout=60s +``` + +Inspect what happened: + +```bash +SB=$(kubectl -n dsh-demo get sandboxclaim dsh-alice -o jsonpath='{.status.sandbox.name}') +echo "alice's sandbox: $SB" +kubectl -n dsh-demo get sandboxes --show-labels +``` + +Three things worth noticing: + +- **The sandbox keeps its pool-generated name** (`dsh-pool-…`), not the claim's name — + adoption re-labels an existing warm sandbox rather than creating one. A platform must always + resolve user → sandbox through `claim.status.sandbox.name`. +- The `agents.x-k8s.io/warm-pool-sandbox` label is **gone** from the adopted sandbox (it left + the pool's inventory), and the pool has already started replenishing back to 2 spares. +- The pod carries the claim's `additionalPodMetadata` label + (`sandbox.users.io/dsh-user=alice`), and + `kubectl -n dsh-demo get sandbox $SB -o jsonpath='{.status.serviceFQDN}'` shows the stable + DNS name the gateway proxies to. + +## 3. Talk to the instance + +```bash +kubectl -n dsh-demo port-forward "$SB" 18081:3080 +# open http://localhost:18081 — DSH Web for alice +``` + +> **Local port note:** DSH Web listens on `3080` **in-cluster**. If `3080` is already in use +> on your workstation (e.g. by a local DSH), forward to a different local port (here `18081`), +> never `3080`. + +## 4. Write state on the PVC + +```bash +kubectl -n dsh-demo exec "$SB" -- sh -c 'echo "remember me" > /home/node/.dsh/marker.txt' +``` + +## 5. Suspend: delete the pod, keep the PVC + Service + +```bash +kubectl -n dsh-demo patch sandbox "$SB" --type merge -p '{"spec":{"operatingMode":"Suspended"}}' +kubectl -n dsh-demo wait --for=delete "pod/$SB" --timeout=120s +kubectl -n dsh-demo get pvc "dsh-home-$SB" # still here +kubectl -n dsh-demo get svc "$SB" # still here +``` + +## 6. Resume and verify state survived + +```bash +kubectl -n dsh-demo patch sandbox "$SB" --type merge -p '{"spec":{"operatingMode":"Running"}}' +kubectl -n dsh-demo wait sandbox "$SB" --for=condition=Ready --timeout=180s +kubectl -n dsh-demo exec "$SB" -- cat /home/node/.dsh/marker.txt # "remember me" +``` + +## 7. Injection policy rejects env-carrying claims + +```bash +kubectl apply -f 40-claim-rejected.yaml +kubectl -n dsh-demo get sandboxclaim dsh-mallory \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' +# EnvVarsInjectionRejected — the claim never bound a sandbox +kubectl delete -f 40-claim-rejected.yaml +``` + +## 8. The gateway: signup, proxy, idle-suspend, wake-on-connect + +Build the gateway image and deploy it: + +```bash +docker build -t dsh-aaas-gateway:demo gateway/ +kind load docker-image dsh-aaas-gateway:demo # (kind clusters) +kubectl apply -f 50-gateway.yaml +kubectl -n dsh-demo rollout status deploy/dsh-gateway --timeout=120s +kubectl -n dsh-demo port-forward svc/dsh-gateway 18080:8080 +``` + +Then: + +```bash +# signup → claim over the warm pool + one-time bearer token +SIGNUP=$(curl -s -X POST http://localhost:18080/users -H 'Content-Type: application/json' -d '{"user":"bob"}') +TOKEN=$(echo "$SIGNUP" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p') + +# proxy to bob's DSH Web (the gateway wakes it if suspended) +curl -s -H "Authorization: Bearer $TOKEN" http://localhost:18080/u/bob/ + +# tokenless is rejected +curl -s -o /dev/null -w '%{http_code}\n' http://localhost:18080/u/bob/ # 401 + +# after IDLE_TIMEOUT (60s) the sweeper suspends bob +curl -s -H "Authorization: Bearer $TOKEN" http://localhost:18080/users/bob # state: Suspended + +# delete bob (cascades sandbox + PVC) +curl -s -X DELETE -H "Authorization: Bearer $TOKEN" http://localhost:18080/users/bob +``` + +## Cost model + +The same economics as the upstream pattern: suspend-when-idle plus warm pools turn an +always-on instance into a sub-second cold-start pause per return-after-absence, at a fraction +of the cost. + +## Cleanup + +```bash +kubectl delete namespace dsh-demo +``` diff --git a/examples/dsh-agents-as-a-service/gateway/Dockerfile b/examples/dsh-agents-as-a-service/gateway/Dockerfile new file mode 100644 index 0000000..f83f0bf --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY gateway.py . +USER 10000 +CMD ["python", "gateway.py"] diff --git a/examples/dsh-agents-as-a-service/gateway/gateway.py b/examples/dsh-agents-as-a-service/gateway/gateway.py new file mode 100644 index 0000000..b52a14f --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/gateway.py @@ -0,0 +1,348 @@ +# Minimal agents-as-a-service gateway for DeepSeek Harness Web (DSH). +# +# A compact, single-file control+data plane adapted from the upstream +# hermes-agents-as-a-service gateway (kubernetes-sigs/agent-sandbox). It turns +# the DSH sandbox resources into a usable multi-user service: +# +# POST /users {"user": "alice"} signup: create a SandboxClaim over the +# warm pool, wait Ready, mint a bearer +# token (only its SHA-256 is stored, as a +# claim annotation) +# GET /users/ derived state: Ready/Waking/Suspended/... +# (requires the user's bearer token) +# DELETE /users/ delete the claim (cascades sandbox + PVC; +# requires the user's bearer token) +# ANY /u// per-user proxy. Requests to a suspended +# sandbox are HELD while it resumes +# (wake-on-connect); everything is proxied +# to the DSH Web UI on :3080. +# +# A background sweeper suspends sandboxes idle for IDLE_TIMEOUT seconds — +# suspend deletes only the pod; the PVC (and the user's state) survives. +# +# Differences from the Hermes original: +# - Single upstream on :3080 (no OpenAI-compatible :8642 / dashboard :9119 +# split, and no platform API key to inject). +# - The original Host header and X-Forwarded-* are forwarded (the gateway's +# own bearer token and any ?token= are still stripped) so DSH generates +# reachable links from the request Host. +# - Requests streaming proxies SSE but does NOT handle WebSocket upgrades; +# a per-user path-based Ingress is the alternative for full WS support. +# +# Deliberately NOT production code: single replica, in-memory idle clock, +# no TLS. + +import hashlib +import os +import re +import secrets +import threading +import time + +import requests +from flask import Flask, Response, jsonify, request +from kubernetes import client, config + +GROUP, VERSION = "extensions.agents.x-k8s.io", "v1beta1" +CORE_GROUP = "agents.x-k8s.io" +NAMESPACE = os.environ.get("NAMESPACE", "dsh-demo") +POOL = os.environ.get("POOL", "dsh-pool") +IDLE_TIMEOUT = int(os.environ.get("IDLE_TIMEOUT", "60")) +WAKE_TIMEOUT = int(os.environ.get("WAKE_TIMEOUT", "120")) +TOKEN_ANNOTATION = "aaas.example.com/token-sha256" +# RFC 1123 label: lowercase alphanumerics and '-', must start/end +# alphanumeric. The claim name is "dsh-", so cap the user part at +# 63 - len("dsh-") to keep the claim a valid Kubernetes object name. +DNS1123_LABEL = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +MAX_USER_LEN = 63 - len("dsh-") + +try: + config.load_incluster_config() +except config.ConfigException: + config.load_kube_config() +crd = client.CustomObjectsApi() + +app = Flask(__name__) +# Cap proxied request bodies: Flask rejects anything larger with 413 before +# request.get_data() would buffer it in memory. +app.config["MAX_CONTENT_LENGTH"] = \ + int(os.environ.get("MAX_BODY_MB", "32")) * 1024 * 1024 +last_activity: dict[str, float] = {} +# Admitted proxy requests per user. While > 0 the sweeper must not suspend: +# a request being served (or still streaming) after IDLE_TIMEOUT is +# activity, not idleness. The sweeper checks-and-suspends under the same +# lock that admission increments under, so the pair is atomic. +in_flight: dict[str, int] = {} +flight_lock = threading.Lock() + + +def end_flight(user: str): + with flight_lock: + in_flight[user] = max(in_flight.get(user, 1) - 1, 0) + last_activity[user] = time.time() + + +def claim_name(user: str) -> str: + return f"dsh-{user}" + + +def get_claim(user: str): + try: + return crd.get_namespaced_custom_object( + GROUP, VERSION, NAMESPACE, "sandboxclaims", claim_name(user)) + except client.ApiException as e: + if e.status == 404: + return None + raise + + +def get_sandbox(name: str): + return crd.get_namespaced_custom_object( + CORE_GROUP, VERSION, NAMESPACE, "sandboxes", name) + + +def sandbox_of(claim): + name = (claim.get("status") or {}).get("sandbox", {}).get("name") + if not name: + return None + try: + return get_sandbox(name) + except client.ApiException as e: + if e.status == 404: # stale claim status, e.g. mid-cascade-delete + return None + raise + + +def condition(obj, ctype: str) -> bool: + for c in (obj.get("status") or {}).get("conditions", []): + if c.get("type") == ctype: + return c.get("status") == "True" + return False + + +def derive_state(sandbox) -> str: + """The state machine a platform shows users (mode x conditions).""" + if sandbox is None: + return "Provisioning" + mode = sandbox["spec"].get("operatingMode", "Running") + if mode == "Suspended": + return "Suspended" if condition(sandbox, "Suspended") else "Suspending" + return "Ready" if condition(sandbox, "Ready") else "Waking" + + +def set_operating_mode(sandbox_name: str, mode: str): + crd.patch_namespaced_custom_object( + CORE_GROUP, VERSION, NAMESPACE, "sandboxes", sandbox_name, + {"spec": {"operatingMode": mode}}) + + +def wait_ready(user: str, timeout: int): + deadline = time.time() + timeout + while time.time() < deadline: + claim = get_claim(user) + sandbox = sandbox_of(claim) if claim else None + if sandbox is not None and derive_state(sandbox) == "Ready": + return sandbox + time.sleep(1) + return None + + +def authorized(claim) -> bool: + # Header-only on purpose: a ?token= query param would leak the bearer + # token into access logs, browser history and Referer headers. + token = request.headers.get("Authorization", "").removeprefix("Bearer ") + want = (claim["metadata"].get("annotations") or {}).get(TOKEN_ANNOTATION, "") + got = hashlib.sha256(token.encode()).hexdigest() + return bool(token) and secrets.compare_digest(got, want) + + +@app.post("/users") +def create_user(): + user = (request.get_json(silent=True) or {}).get("user", "") + if not isinstance(user, str) or not DNS1123_LABEL.fullmatch(user) \ + or len(user) > MAX_USER_LEN: + return jsonify(error="body must be {'user': ''}"), 400 + if get_claim(user) is not None: + return jsonify(error="user exists"), 409 + token = secrets.token_urlsafe(32) + # The claim IS the user record: warmPoolRef only (env/VCTs would be + # rejected by the template policies), token hash as an annotation. + try: + crd.create_namespaced_custom_object( + GROUP, VERSION, NAMESPACE, "sandboxclaims", { + "apiVersion": f"{GROUP}/{VERSION}", + "kind": "SandboxClaim", + "metadata": { + "name": claim_name(user), + "annotations": { + TOKEN_ANNOTATION: hashlib.sha256(token.encode()).hexdigest()}, + }, + "spec": { + "warmPoolRef": {"name": POOL}, + "additionalPodMetadata": { + "labels": {"sandbox.users.io/dsh-user": user}}, + }, + }) + except client.ApiException as e: + if e.status == 409: # lost a concurrent-signup race for this user + return jsonify(error="user exists"), 409 + raise + sandbox = wait_ready(user, WAKE_TIMEOUT) + last_activity[user] = time.time() + return jsonify( + user=user, + token=token, # shown once; only the hash is stored + state=derive_state(sandbox), + sandbox=(sandbox or {}).get("metadata", {}).get("name"), + ), 201 + + +@app.get("/users/") +def get_user(user): + claim = get_claim(user) + if claim is None: + return jsonify(error="not found"), 404 + if not authorized(claim): # only the token holder may read their state + return jsonify(error="unauthorized"), 401 + sandbox = sandbox_of(claim) + return jsonify(user=user, state=derive_state(sandbox), + sandbox=(sandbox or {}).get("metadata", {}).get("name")) + + +@app.delete("/users/") +def delete_user(user): + claim = get_claim(user) + if claim is None: + return jsonify(error="not found"), 404 + if not authorized(claim): # deletion cascades the PVC: token required + return jsonify(error="unauthorized"), 401 + try: + crd.delete_namespaced_custom_object( + GROUP, VERSION, NAMESPACE, "sandboxclaims", claim_name(user)) + except client.ApiException as e: + if e.status != 404: # already gone => idempotent success + raise + last_activity.pop(user, None) + return jsonify(user=user, note="claim deleted; sandbox and PVC cascade") + + +@app.route("/u//", defaults={"path": ""}, + methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) +@app.route("/u//", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) +def proxy(user, path): + claim = get_claim(user) + if claim is None: + return jsonify(error="unknown user"), 404 + if not authorized(claim): + return jsonify(error="unauthorized"), 401 + + # Reserve activity the moment the request is admitted — BEFORE the + # sandbox is resolved or woken — so the sweeper can never suspend + # between admission and the upstream dial. Every early return releases + # the reservation; the streaming path hands it to relay(), which + # releases it when the response finishes (or the client disconnects). + with flight_lock: + in_flight[user] = in_flight.get(user, 0) + 1 + last_activity[user] = time.time() + streaming = False + try: + sandbox = sandbox_of(claim) + if sandbox is None: + return jsonify(error="sandbox not provisioned yet"), 503 + if derive_state(sandbox) != "Ready": + # Wake-on-connect: flip the mode (safe in ANY state, including + # mid-suspension) and HOLD the request until Ready. + set_operating_mode(sandbox["metadata"]["name"], "Running") + sandbox = wait_ready(user, WAKE_TIMEOUT) + if sandbox is None: + return jsonify(error="DSH is waking up, retry shortly"), \ + 503, {"Retry-After": "10"} + last_activity[user] = time.time() + + fqdn = (sandbox.get("status") or {}).get("serviceFQDN") + # Forward the original Host + X-Forwarded-* so DSH generates + # reachable links; still never forward the gateway's bearer token or + # a stray ?token= upstream where the app (or its logs) would see it. + headers = {k: v for k, v in request.headers + if k.lower() != "authorization"} + headers.setdefault("X-Forwarded-For", request.remote_addr or "") + headers.setdefault("X-Forwarded-Proto", request.scheme) + headers.setdefault("X-Forwarded-Host", request.host) + params = [(k, v) for k, v in request.args.items(multi=True) + if k != "token"] + upstream = f"http://{fqdn}:3080/{path}" + + # The pod can be Ready a beat before the server binds; retry the + # dial. + for attempt in range(5): + try: + r = requests.request( + request.method, upstream, params=params, + data=request.get_data(), headers=headers, + stream=True, timeout=300) + break + except requests.ConnectionError: + if attempt == 4: + return jsonify(error="upstream unavailable"), 502 + time.sleep(1) + + def relay(): + try: + yield from r.iter_content(chunk_size=None) + finally: # runs on stream completion AND client disconnect + r.close() # release the upstream connection either way + end_flight(user) + + streaming = True # relay() now owns the in-flight reservation + return Response( + relay(), + status=r.status_code, + headers=[(k, v) for k, v in r.headers.items() + if k.lower() not in ("transfer-encoding", "connection")]) + finally: + if not streaming: + end_flight(user) + + +def idle_sweeper(): + """Suspend sandboxes idle for IDLE_TIMEOUT (pod deleted, PVC survives).""" + while True: + time.sleep(15) + try: + claims = crd.list_namespaced_custom_object( + GROUP, VERSION, NAMESPACE, "sandboxclaims")["items"] + for c in claims: + user = c["metadata"]["name"].removeprefix("dsh-") + try: + sandbox = sandbox_of(c) + except client.ApiException: + continue + if sandbox is None or derive_state(sandbox) != "Ready": + continue + # Check-and-suspend atomically with proxy admission (which + # increments in_flight under this same lock). + with flight_lock: + if in_flight.get(user, 0) > 0: + continue # requests in flight => the user is active + idle = time.time() - \ + last_activity.setdefault(user, time.time()) + if idle < IDLE_TIMEOUT: + continue + print(f"suspending {user} (idle {int(idle)}s)", + flush=True) + set_operating_mode(sandbox["metadata"]["name"], + "Suspended") + except Exception as e: + # A transient API error must not kill this daemon thread — that + # would silently stop idle suspension for good. + print(f"sweeper: transient error, retrying next cycle: {e}", + flush=True) + + +if __name__ == "__main__": + threading.Thread(target=idle_sweeper, daemon=True).start() + # threaded=True so one user's wake-on-connect (which can hold a request + # for up to WAKE_TIMEOUT) doesn't block everyone else. Still Werkzeug's + # dev server — a real deployment would sit behind gunicorn or similar. + app.run(host="0.0.0.0", port=8080, threaded=True) diff --git a/examples/dsh-agents-as-a-service/gateway/requirements.txt b/examples/dsh-agents-as-a-service/gateway/requirements.txt new file mode 100644 index 0000000..ad59624 --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/requirements.txt @@ -0,0 +1,3 @@ +flask==3.1.* +requests==2.32.* +kubernetes==31.0.* diff --git a/examples/dsh-agents-as-a-service/gateway/test_gateway.py b/examples/dsh-agents-as-a-service/gateway/test_gateway.py new file mode 100644 index 0000000..eba2a68 --- /dev/null +++ b/examples/dsh-agents-as-a-service/gateway/test_gateway.py @@ -0,0 +1,382 @@ +""" +Unit tests for the dsh-agents-as-a-service gateway example. + +gateway.py runs as the control+data plane in front of agent-sandbox; these +tests cover its security/parsing logic (bearer-token auth, DNS-1123 user +validation, the mode x conditions state machine) and the /users and +idle-sweeper decision logic, with the Kubernetes CustomObjectsApi mocked. +No kind cluster or container build is needed. + +gateway.py isn't a valid module name to import as a package member from a +test file that also wants a fresh module per test (its top level calls +config.load_incluster_config()/load_kube_config(), both of which would blow +up or reach a real cluster), so each test loads its own copy by file path +via the `gw` fixture, mirroring examples/hpa-swp-scaling/test_create_claim.py. +""" + +import hashlib +import importlib.util +import json +import os +import time +from types import SimpleNamespace +from unittest import mock + +import pytest +from kubernetes import client as k8s_client +from kubernetes.config import ConfigException + +GATEWAY_PATH = os.path.join(os.path.dirname(__file__), "gateway.py") + + +class _StopLoop(Exception): + """Sentinel used to break idle_sweeper()'s `while True` after one pass.""" + + +def _load_gateway(monkeypatch, mock_crd): + monkeypatch.setenv("NAMESPACE", "test-ns") + monkeypatch.setenv("POOL", "test-pool") + monkeypatch.setenv("IDLE_TIMEOUT", "60") + monkeypatch.setenv("WAKE_TIMEOUT", "120") + + with mock.patch("kubernetes.config.load_incluster_config", + side_effect=ConfigException("not running in a cluster")), \ + mock.patch("kubernetes.config.load_kube_config"), \ + mock.patch("kubernetes.client.CustomObjectsApi", return_value=mock_crd): + spec = importlib.util.spec_from_file_location("dsh_gateway", GATEWAY_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def gw(monkeypatch): + """A freshly loaded gateway module (own last_activity/in_flight state) + with its Kubernetes CustomObjectsApi mocked, plus a Flask test client.""" + mock_crd = mock.MagicMock() + module = _load_gateway(monkeypatch, mock_crd) + module.app.testing = True # propagate unhandled exceptions instead of a bare 500 + return SimpleNamespace(module=module, crd=mock_crd, client=module.app.test_client()) + + +def crd_get_side_effect(claims=None, sandboxes=None): + """Builds a get_namespaced_custom_object side_effect that serves + "sandboxclaims" from `claims` and "sandboxes" from `sandboxes` (both + {name: obj_or_None} maps), raising a 404 ApiException for anything + missing -- mirroring how get_claim()/get_sandbox() interpret 404.""" + claims = claims or {} + sandboxes = sandboxes or {} + + def _get(group, version, namespace, plural, name): + if plural == "sandboxclaims": + table = claims + elif plural == "sandboxes": + table = sandboxes + else: + raise AssertionError(f"unexpected plural {plural!r}") + obj = table.get(name) + if obj is None: + raise k8s_client.ApiException(status=404) + return obj + + return _get + + +# --- condition() --------------------------------------------------------- + +def test_condition_true_when_status_matches(gw): + obj = {"status": {"conditions": [{"type": "Ready", "status": "True"}]}} + assert gw.module.condition(obj, "Ready") is True + + +def test_condition_false_when_status_is_false(gw): + obj = {"status": {"conditions": [{"type": "Ready", "status": "False"}]}} + assert gw.module.condition(obj, "Ready") is False + + +def test_condition_false_when_type_absent(gw): + obj = {"status": {"conditions": [{"type": "Suspended", "status": "True"}]}} + assert gw.module.condition(obj, "Ready") is False + + +def test_condition_false_when_no_status(gw): + assert gw.module.condition({}, "Ready") is False + + +# --- derive_state() -------------------------------------------------------- + +def test_derive_state_provisioning_when_sandbox_missing(gw): + assert gw.module.derive_state(None) == "Provisioning" + + +def test_derive_state_ready(gw): + sandbox = {"spec": {}, "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + assert gw.module.derive_state(sandbox) == "Ready" + + +def test_derive_state_waking_when_not_yet_ready(gw): + sandbox = {"spec": {"operatingMode": "Running"}, "status": {"conditions": []}} + assert gw.module.derive_state(sandbox) == "Waking" + + +def test_derive_state_suspended(gw): + sandbox = {"spec": {"operatingMode": "Suspended"}, + "status": {"conditions": [{"type": "Suspended", "status": "True"}]}} + assert gw.module.derive_state(sandbox) == "Suspended" + + +def test_derive_state_suspending_while_mode_flipped_but_not_settled(gw): + sandbox = {"spec": {"operatingMode": "Suspended"}, "status": {"conditions": []}} + assert gw.module.derive_state(sandbox) == "Suspending" + + +# --- authorized() ---------------------------------------------------------- + +def _claim_for_token(module, token): + return {"metadata": {"annotations": { + module.TOKEN_ANNOTATION: hashlib.sha256(token.encode()).hexdigest()}}} + + +def test_authorized_accepts_correct_bearer_token(gw): + claim = _claim_for_token(gw.module, "s3cret") + with gw.module.app.test_request_context(headers={"Authorization": "Bearer s3cret"}): + assert gw.module.authorized(claim) is True + + +def test_authorized_rejects_wrong_token(gw): + claim = _claim_for_token(gw.module, "s3cret") + with gw.module.app.test_request_context(headers={"Authorization": "Bearer wrong"}): + assert gw.module.authorized(claim) is False + + +def test_authorized_rejects_missing_header(gw): + claim = _claim_for_token(gw.module, "s3cret") + with gw.module.app.test_request_context(): + assert gw.module.authorized(claim) is False + + +def test_authorized_rejects_empty_bearer_token(gw): + claim = _claim_for_token(gw.module, "") + with gw.module.app.test_request_context(headers={"Authorization": "Bearer "}): + # bool(token) guards against an empty token matching an empty/unset + # annotation hash. + assert gw.module.authorized(claim) is False + + +# --- POST /users (create_user) --------------------------------------------- + +@pytest.mark.parametrize("bad_user", ["Alice", "-alice", "alice-", "", "a_b", 123, None]) +def test_create_user_rejects_invalid_dns1123_label(gw, bad_user): + response = gw.client.post("/users", json={"user": bad_user}) + assert response.status_code == 400 + gw.crd.create_namespaced_custom_object.assert_not_called() + + +def test_create_user_rejects_label_over_max_len(gw): + too_long = "a" * (gw.module.MAX_USER_LEN + 1) + response = gw.client.post("/users", json={"user": too_long}) + assert response.status_code == 400 + gw.crd.create_namespaced_custom_object.assert_not_called() + + +def test_create_user_conflict_when_claim_already_exists(gw): + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": {"metadata": {"name": "dsh-alice"}}}) + + response = gw.client.post("/users", json={"user": "alice"}) + + assert response.status_code == 409 + gw.crd.create_namespaced_custom_object.assert_not_called() + + +def test_create_user_conflict_on_concurrent_signup_race(gw): + gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) + gw.crd.create_namespaced_custom_object.side_effect = k8s_client.ApiException(status=409) + + response = gw.client.post("/users", json={"user": "alice"}) + + assert response.status_code == 409 + + +def test_create_user_success_stores_only_token_hash(gw, monkeypatch): + gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) + ready_sandbox = { + "metadata": {"name": "dsh-alice-sbx"}, + "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + } + # wait_ready() polls with time.sleep(1); stub it out rather than racing + # a real poll loop in a unit test. + monkeypatch.setattr(gw.module, "wait_ready", lambda user, timeout: ready_sandbox) + + response = gw.client.post("/users", json={"user": "alice"}) + + assert response.status_code == 201 + body = response.get_json() + assert body["user"] == "alice" + assert body["state"] == "Ready" + assert body["sandbox"] == "dsh-alice-sbx" + assert body["token"] # shown once in the response... + + gw.crd.create_namespaced_custom_object.assert_called_once() + claim_body = gw.crd.create_namespaced_custom_object.call_args.args[4] + assert claim_body["metadata"]["name"] == "dsh-alice" + # ...but only its SHA-256 is ever sent to the API server. + want_hash = hashlib.sha256(body["token"].encode()).hexdigest() + assert claim_body["metadata"]["annotations"][gw.module.TOKEN_ANNOTATION] == want_hash + assert body["token"] not in json.dumps(claim_body) + assert claim_body["spec"]["warmPoolRef"]["name"] == "test-pool" + assert claim_body["spec"]["additionalPodMetadata"]["labels"]["sandbox.users.io/dsh-user"] == "alice" + + +# --- GET /users/ and DELETE /users/ ----------------------------- + +def test_get_user_not_found(gw): + gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) + response = gw.client.get("/users/alice") + assert response.status_code == 404 + + +def test_get_user_requires_valid_token(gw): + claim = _claim_for_token(gw.module, "s3cret") + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}) + + response = gw.client.get("/users/alice", headers={"Authorization": "Bearer wrong"}) + + assert response.status_code == 401 + + +def test_get_user_returns_state_when_authorized(gw): + claim = _claim_for_token(gw.module, "s3cret") + sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + claim["status"] = {"sandbox": {"name": "dsh-alice-sbx"}} + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}, sandboxes={"dsh-alice-sbx": sandbox}) + + response = gw.client.get("/users/alice", headers={"Authorization": "Bearer s3cret"}) + + assert response.status_code == 200 + assert response.get_json()["state"] == "Ready" + + +def test_delete_user_not_found(gw): + gw.crd.get_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) + response = gw.client.delete("/users/alice") + assert response.status_code == 404 + + +def test_delete_user_requires_valid_token(gw): + claim = _claim_for_token(gw.module, "s3cret") + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}) + + response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer wrong"}) + + assert response.status_code == 401 + gw.crd.delete_namespaced_custom_object.assert_not_called() + + +def test_delete_user_cascades_and_clears_activity(gw): + claim = _claim_for_token(gw.module, "s3cret") + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}) + gw.module.last_activity["alice"] = time.time() + + response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) + + assert response.status_code == 200 + gw.crd.delete_namespaced_custom_object.assert_called_once() + assert "alice" not in gw.module.last_activity + + +def test_delete_user_already_gone_is_idempotent(gw): + claim = _claim_for_token(gw.module, "s3cret") + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}) + gw.crd.delete_namespaced_custom_object.side_effect = k8s_client.ApiException(status=404) + + response = gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) + + assert response.status_code == 200 + + +def test_delete_user_unexpected_api_error_propagates(gw): + claim = _claim_for_token(gw.module, "s3cret") + gw.crd.get_namespaced_custom_object.side_effect = crd_get_side_effect( + claims={"dsh-alice": claim}) + gw.crd.delete_namespaced_custom_object.side_effect = k8s_client.ApiException(status=500) + + with pytest.raises(k8s_client.ApiException): + gw.client.delete("/users/alice", headers={"Authorization": "Bearer s3cret"}) + + +# --- idle_sweeper() ---------------------------------------------------------- + +def _run_one_sweep(gw): + # time.sleep(15) runs before the try/except in idle_sweeper's loop body, + # so the first call must succeed (to let one pass execute) and only the + # second call breaks out. + with mock.patch("time.sleep", side_effect=[None, _StopLoop()]): + with pytest.raises(_StopLoop): + gw.module.idle_sweeper() + + +def test_idle_sweeper_suspends_idle_ready_sandbox(gw): + sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + claim_item = {"metadata": {"name": "dsh-alice"}, + "status": {"sandbox": {"name": "dsh-alice-sbx"}}} + gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} + gw.crd.get_namespaced_custom_object.return_value = sandbox + gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 + + _run_one_sweep(gw) + + gw.crd.patch_namespaced_custom_object.assert_called_once() + args = gw.crd.patch_namespaced_custom_object.call_args.args + assert args[4] == "dsh-alice-sbx" + assert args[5] == {"spec": {"operatingMode": "Suspended"}} + + +def test_idle_sweeper_skips_user_with_in_flight_request(gw): + sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + claim_item = {"metadata": {"name": "dsh-alice"}, + "status": {"sandbox": {"name": "dsh-alice-sbx"}}} + gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} + gw.crd.get_namespaced_custom_object.return_value = sandbox + gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 + gw.module.in_flight["alice"] = 1 + + _run_one_sweep(gw) + + gw.crd.patch_namespaced_custom_object.assert_not_called() + + +def test_idle_sweeper_skips_recently_active_user(gw): + sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + claim_item = {"metadata": {"name": "dsh-alice"}, + "status": {"sandbox": {"name": "dsh-alice-sbx"}}} + gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} + gw.crd.get_namespaced_custom_object.return_value = sandbox + gw.module.last_activity["alice"] = time.time() + + _run_one_sweep(gw) + + gw.crd.patch_namespaced_custom_object.assert_not_called() + + +def test_idle_sweeper_skips_sandbox_not_ready(gw): + sandbox = {"metadata": {"name": "dsh-alice-sbx"}, "spec": {}, "status": {"conditions": []}} + claim_item = {"metadata": {"name": "dsh-alice"}, + "status": {"sandbox": {"name": "dsh-alice-sbx"}}} + gw.crd.list_namespaced_custom_object.return_value = {"items": [claim_item]} + gw.crd.get_namespaced_custom_object.return_value = sandbox + gw.module.last_activity["alice"] = time.time() - gw.module.IDLE_TIMEOUT - 10 + + _run_one_sweep(gw) + + gw.crd.patch_namespaced_custom_object.assert_not_called() diff --git a/examples/dsh-agents-as-a-service/run-test-kind.sh b/examples/dsh-agents-as-a-service/run-test-kind.sh new file mode 100755 index 0000000..d502f87 --- /dev/null +++ b/examples/dsh-agents-as-a-service/run-test-kind.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Smoke test for the dsh-agents-as-a-service example: runs the full README +# walkthrough non-interactively against the current kubectl context. +# Expects agent-sandbox (with extensions) already installed. +# +# NOTE: the only local port bound is the gateway port-forward on 18080. +# The DSH Web container port (3080) is in-cluster only and is never bound +# on the local host. +set -euo pipefail +cd "$(dirname "$0")" + +NS=dsh-demo +pass() { echo "PASS: $1"; } +fail() { echo "FAIL: $1" >&2; exit 1; } +cleanup() { kubectl delete namespace "$NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true; } +trap cleanup EXIT + +echo "=== 1. install prereqs (throwaway key) + template + warm pool" +kubectl apply -f 00-prereqs.yaml +# The provider key is never committed; generate a throwaway value. +kubectl -n "$NS" delete secret dsh-provider-keys --ignore-not-found >/dev/null +kubectl -n "$NS" create secret generic dsh-provider-keys \ + --from-literal=DEEPSEEK_API_KEY="sk-test-$(openssl rand -hex 16)" +kubectl apply -f 10-sandboxtemplate.yaml -f 20-sandboxwarmpool.yaml +for _ in $(seq 1 120); do + READY=$(kubectl -n "$NS" get sandboxes -l agents.x-k8s.io/warm-pool-sandbox \ + -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' | grep -c True || true) + [ "$READY" -ge 2 ] && break + sleep 5 +done +[ "${READY:-0}" -ge 2 ] || fail "warm pool did not reach 2 Ready spares" +pass "warm pool: 2 spares Ready" + +echo "=== 2. signup via claim (warm adoption)" +T0=$(date +%s) +kubectl apply -f 30-claim-alice.yaml +kubectl -n "$NS" wait sandboxclaim dsh-alice --for=condition=Ready --timeout=60s +ELAPSED=$(( $(date +%s) - T0 )) +SB=$(kubectl -n "$NS" get sandboxclaim dsh-alice -o jsonpath='{.status.sandbox.name}') +case "$SB" in dsh-pool-*) ;; *) fail "expected warm adoption, got sandbox '$SB'";; esac +pass "claim Ready in ${ELAPSED}s, adopted $SB" + +echo "=== 3. write state on the PVC" +POD="$SB" +kubectl -n "$NS" exec "$POD" -- sh -c 'echo "remember me" > /home/node/.dsh/marker.txt' +pass "marker written in $POD" + +echo "=== 4. suspend (pod deleted, PVC + Service retained)" +kubectl -n "$NS" patch sandbox "$SB" --type merge -p '{"spec":{"operatingMode":"Suspended"}}' +kubectl -n "$NS" wait --for=delete "pod/$POD" --timeout=120s +kubectl -n "$NS" get pvc "dsh-home-$SB" >/dev/null || fail "PVC was not retained" +kubectl -n "$NS" get svc "$SB" >/dev/null || fail "Service was not retained" +pass "suspended: pod gone, PVC and Service retained" + +echo "=== 5. resume and verify state survived" +kubectl -n "$NS" patch sandbox "$SB" --type merge -p '{"spec":{"operatingMode":"Running"}}' +kubectl -n "$NS" wait sandbox "$SB" --for=condition=Ready --timeout=180s +POD="$SB" +MARKER=$(kubectl -n "$NS" exec "$POD" -- cat /home/node/.dsh/marker.txt) +[ "$MARKER" = "remember me" ] || fail "marker did not survive suspend/resume" +pass "resumed: state survived on the reattached PVC" + +echo "=== 6. injection policy rejects env-carrying claims" +kubectl apply -f 40-claim-rejected.yaml +# Wait for the POSITIVE rejection signal, not merely "not bound yet". +REASON="" +for _ in $(seq 1 12); do + REASON=$(kubectl -n "$NS" get sandboxclaim dsh-mallory \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true) + [ "$REASON" = "EnvVarsInjectionRejected" ] && break + sleep 5 +done +[ "$REASON" = "EnvVarsInjectionRejected" ] \ + || fail "claim was not rejected (Ready reason='$REASON')" +BOUND=$(kubectl -n "$NS" get sandboxclaim dsh-mallory -o jsonpath='{.status.sandbox.name}' 2>/dev/null || true) +[ -z "$BOUND" ] || fail "env-carrying claim unexpectedly bound sandbox '$BOUND'" +kubectl delete -f 40-claim-rejected.yaml >/dev/null +pass "claim rejected with EnvVarsInjectionRejected and never bound" + +echo "=== 7. gateway: signup, proxy, idle-suspend, wake-on-connect" +if command -v docker >/dev/null && command -v kind >/dev/null; then + KIND_CLUSTER="${KIND_CLUSTER:-$(kubectl config current-context | sed 's/^kind-//')}" + docker build -q -t dsh-aaas-gateway:demo gateway/ >/dev/null + kind load docker-image dsh-aaas-gateway:demo --name "$KIND_CLUSTER" >/dev/null + kubectl apply -f 50-gateway.yaml + kubectl -n "$NS" rollout status deploy/dsh-gateway --timeout=120s + # Local port-forward uses 18080 (NOT 3080, which stays in-cluster). + kubectl -n "$NS" port-forward svc/dsh-gateway 18080:8080 >/dev/null 2>&1 & + PF=$! + sleep 3 + GW=localhost:18080 + + SIGNUP=$(curl -s -X POST "http://$GW/users" -H 'Content-Type: application/json' -d '{"user":"bob"}') + TOKEN=$(echo "$SIGNUP" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p') + [ -n "$TOKEN" ] || fail "signup returned no token: $SIGNUP" + echo "$SIGNUP" | grep -q '"state":"Ready"' || fail "signup not Ready: $SIGNUP" + pass "gateway signup: bob Ready with one-time token" + + curl -sf -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \ + || fail "proxied web call failed" + curl -s -o /dev/null -w '%{http_code}' "http://$GW/u/bob/" | grep -q 401 \ + || fail "tokenless request was not rejected" + pass "gateway proxy: authed request 200, tokenless 401" + + echo "waiting for idle sweeper (~75s)..." + for _ in $(seq 1 30); do + STATE=$(curl -s -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" \ + | sed -n 's/.*"state":"\([^"]*\)".*/\1/p') + [ "$STATE" = "Suspended" ] && break + sleep 5 + done + [ "$STATE" = "Suspended" ] || fail "bob was not idle-suspended (state=$STATE)" + pass "idle sweeper suspended bob" + + T0=$(date +%s) + curl -sf -m 150 -H "Authorization: Bearer $TOKEN" "http://$GW/u/bob/" >/dev/null \ + || fail "wake-on-connect request failed" + pass "wake-on-connect: request held and served after $(( $(date +%s) - T0 ))s" + + curl -sf -X DELETE -H "Authorization: Bearer $TOKEN" "http://$GW/users/bob" >/dev/null \ + || fail "gateway user deletion failed" + kill "$PF" 2>/dev/null || true +elif [ "${SKIP_GATEWAY:-}" = "1" ]; then + echo "WARNING: gateway checks SKIPPED (SKIP_GATEWAY=1) — signup, auth," >&2 + echo "WARNING: proxy, idle-suspend and wake-on-connect were NOT validated" >&2 + GATEWAY_SKIPPED=1 +else + fail "docker + kind are required for the gateway checks (set SKIP_GATEWAY=1 to run only the resource-layer checks)" +fi + +echo "=== 8. cascade delete" +kubectl -n "$NS" delete sandboxclaim dsh-alice +for _ in $(seq 1 24); do + kubectl -n "$NS" get sandbox "$SB" >/dev/null 2>&1 || break + sleep 5 +done +kubectl -n "$NS" get sandbox "$SB" >/dev/null 2>&1 && fail "sandbox not garbage-collected" +for _ in $(seq 1 24); do + kubectl -n "$NS" get pvc "dsh-home-$SB" >/dev/null 2>&1 || break + sleep 5 +done +kubectl -n "$NS" get pvc "dsh-home-$SB" >/dev/null 2>&1 && fail "PVC not garbage-collected" +pass "claim delete cascaded: sandbox + PVC gone" + +echo +if [ -n "${GATEWAY_SKIPPED:-}" ]; then + echo "Resource-layer checks passed; gateway checks were SKIPPED." +else + echo "All checks passed." +fi