mirror of
https://github.com/sredevopsorg/deepseek-kubernetes-sandbox.git
synced 2026-11-03 01:13:24 +00:00
367294b230
/workspace - the DSH image's WORKDIR, where agents keep files - was plain container storage, so every suspend/resume or pod replacement silently threw agent work away while DSH state survived on the PVC. Mount the same per-sandbox `dsh-home` PVC a second time at /workspace with `subPath: workspace`. One PVC, no extra provisioning, and the subPath keeps DSH state (sessions, config and .credentials.yaml at the volume root) out of the agent-visible tree; mounting the volume root at /workspace would have exposed the credential file. The kubelet creates the missing subPath directory itself and chmods it to the volume's fsGroup permissions, and the volume root's setgid bit makes the new directory inherit gid 1000, so UID 1000 can write with no init container and no root. The rare interrupted-kubelet case that leaves 0755 is documented with a recovery procedure (kubernetes#142464) rather than papered over. - e2e-kind: write a /workspace marker alongside the DSH_HOME marker, assert the mount is the dsh-home subPath, and verify both survive suspend/resume. - Docs: README (overview, steps 4-6, troubleshooting row), architecture (PVC diagram, persistence, component table), main-image working-directory row, platform runbook PVC row. Verified: make check (14 manifests), make lint, plus YAML assertions on the two mounts and the unchanged volumeClaimTemplates. Co-authored-by: SREDevOpsAgents <agents@sredevops.org>
171 lines
8.0 KiB
YAML
171 lines
8.0 KiB
YAML
# The reusable DSH Web blueprint. Warm pools stamp pod env BEFORE any user
|
|
# exists, so per-user secrets cannot live in pod env; the shared provider keys
|
|
# are injected via envFrom, and claims that would bypass the warm pool are
|
|
# rejected outright (see 40-claim-rejected.yaml).
|
|
#
|
|
# A stripped-down single-tenant variant of this template (no warm pool, no
|
|
# policy enforcement) lives in examples/minimal-single-tenant/.
|
|
apiVersion: extensions.agents.x-k8s.io/v1beta1
|
|
kind: SandboxTemplate
|
|
metadata:
|
|
name: dsh-web
|
|
namespace: dsh-demo
|
|
spec:
|
|
# Claims must not inject env or volumeClaimTemplates: with Disallowed, such
|
|
# claims are REJECTED instead of silently taking the cold (non-warm-pool)
|
|
# provisioning path. Demonstrated in step 6.
|
|
envVarsInjectionPolicy: Disallowed
|
|
volumeClaimTemplatesPolicy: Disallowed
|
|
|
|
# Per-sandbox headless Service -> stable DNS (sandbox.status.serviceFQDN).
|
|
# The gateway routes each user to their DSH Web instance by this name.
|
|
service: true
|
|
|
|
# The example leaves NetworkPolicy to an optional, opt-in manifest
|
|
# (manifests/optional/network-policy/): the base walkthrough should not
|
|
# require a policy-capable CNI, and the controller's Managed secure default
|
|
# only allows the sandbox-router, which this example does not deploy. If you
|
|
# switch to Managed, define spec.networkPolicy explicitly or the gateway will
|
|
# be blocked. See docs/optional-hardening.md.
|
|
networkPolicyManagement: Unmanaged
|
|
|
|
# OPTIONAL: replace a Failed sandbox Pod instead of parking the sandbox.
|
|
# The default, Retain, leaves the Failed Pod in place and surfaces
|
|
# Finished=True on the Sandbox until a human intervenes. Recreate deletes the
|
|
# Failed Pod and starts a replacement with the same Sandbox identity and
|
|
# Sandbox-owned PVCs, backing off (5s doubling to 5m) on repeated failures.
|
|
# Requires agent-sandbox v1.0.8+; existing Sandboxes keep their own stamped
|
|
# policy, so patch them separately. See docs/optional-hardening.md.
|
|
# failedPodPolicy: Recreate
|
|
|
|
podTemplate:
|
|
spec:
|
|
# DSH Web is multi-arch (amd64 + arm64); no arch nodeSelector is needed.
|
|
#
|
|
# OPTIONAL: run the sandbox under gVisor. Requires a RuntimeClass named
|
|
# "gvisor" and nodes that can run it; gVisor breaks `kubectl port-forward`
|
|
# into the sandbox (use the gateway). Existing Sandboxes snapshot their
|
|
# pod template, so enabling it later requires patching/recreating them.
|
|
# See docs/optional-hardening.md.
|
|
# runtimeClassName: gvisor
|
|
#
|
|
# OPTIONAL: pin sandboxes to a dedicated node pool (for example a gVisor
|
|
# pool). Replace this placeholder with a label your nodes actually carry;
|
|
# add a matching toleration if the pool is tainted. Active
|
|
# cluster-specific scheduling belongs in a local patch, not here.
|
|
# nodeSelector:
|
|
# dsh.example.com/node-pool: dsh
|
|
restartPolicy: "OnFailure"
|
|
securityContext:
|
|
runAsUser: 1000 # the 'node' user in the DSH image
|
|
runAsGroup: 1000
|
|
fsGroup: 1000
|
|
runAsNonRoot: true
|
|
# DSH's credentials plugin refuses to start when .credentials.yaml has
|
|
# any group/other permission bit (it creates the file 0600). The
|
|
# default kubelet fsGroup policy ("Always") recursively ORs 0660 into
|
|
# every file each time the pod starts, so a resumed sandbox comes back
|
|
# with 0660 and crash-loops before the gateway can proxy to it.
|
|
# OnRootMismatch skips that walk once the volume root already carries
|
|
# the fsGroup ownership/mode applied on the first mount.
|
|
# Keep in sync with the minimal variant:
|
|
# examples/minimal-single-tenant/manifests/sandbox-template.yaml.
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
containers:
|
|
- name: dsh-web
|
|
image: ghcr.io/sredevopsorg/deepseek-kubernetes-sandbox:dsh-v0.2.1-rc.5
|
|
imagePullPolicy: IfNotPresent
|
|
ports:
|
|
- name: web
|
|
containerPort: 3080
|
|
envFrom:
|
|
- secretRef:
|
|
name: dsh-provider-keys
|
|
# Optional extra provider keys (OpenRouter, Gemini, NVIDIA, ...).
|
|
# The Secret may be absent; every key in it becomes an env var.
|
|
# Create it before the warm pool, because pod env is stamped at
|
|
# pod creation. See docs/optional-hardening.md.
|
|
- secretRef:
|
|
name: dsh-provider-keys-extra
|
|
optional: true
|
|
env:
|
|
# Empty => DSH derives links from the request Host, which the
|
|
# gateway forwards (plus X-Forwarded-*). Point this at the
|
|
# gateway base URL if you want hard-coded absolute links instead.
|
|
- name: DSH_PUBLIC_URL
|
|
value: ""
|
|
# DSH Host/Origin-fences /api. The gateway forwards the browser's
|
|
# Host, so DSH must trust the public gateway host. Empty is right
|
|
# for port-forward/localhost; for either exposure variant set this
|
|
# to the public host (e.g. dsh.example.com for
|
|
# manifests/exposure/ingress/60-ingress.yaml, or
|
|
# dsh-demo.example.com for the gateway-api-sso/ HTTPRoute).
|
|
- name: DSH_TRUSTED_HOSTS
|
|
value: ""
|
|
# Probes target /favicon.svg: DSH gates / behind its launch-token
|
|
# cookie and answers 401 to an unauthenticated GET, which would
|
|
# otherwise fail readiness and then get the pod killed by liveness.
|
|
# /favicon.svg is served without auth and is the cheapest liveness
|
|
# signal that proves the HTTP server is actually serving requests.
|
|
#
|
|
# Gate liveness/readiness behind a startup probe: a cold start on a
|
|
# small node takes tens of seconds to load plugins and start serving
|
|
# (a resumed sandbox starts cold too). Without it the kubelet can kill
|
|
# DSH mid-startup, and a SIGTERM'd DSH exits 0, which the sandbox
|
|
# controller records as finished. 24 x 5s = 120s, matching the
|
|
# gateway's WAKE_TIMEOUT.
|
|
startupProbe:
|
|
httpGet:
|
|
path: /favicon.svg
|
|
port: 3080
|
|
periodSeconds: 5
|
|
failureThreshold: 24
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /favicon.svg
|
|
port: 3080
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
failureThreshold: 30
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /favicon.svg
|
|
port: 3080
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 15
|
|
failureThreshold: 3
|
|
# The container only needs to serve HTTP; drop every capability and
|
|
# disallow privilege escalation. DSH writes only to its PVC-backed
|
|
# directories (DSH_HOME and the agent workspace).
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests: {cpu: 250m, memory: 512Mi}
|
|
limits: {cpu: "1", memory: 1Gi}
|
|
volumeMounts:
|
|
- name: dsh-home
|
|
mountPath: /home/node/.dsh # DSH_HOME: sessions, config, state
|
|
# The agent workspace (the image's WORKDIR) is PVC-backed too, but
|
|
# under its own subPath so DSH state - including .credentials.yaml -
|
|
# never shows up as workspace data. Kubelet creates workspace/ on
|
|
# first mount with the volume's fsGroup permissions (0775, setgid
|
|
# inherited from the volume root), so UID 1000 can write; no init
|
|
# container is needed. If the kubelet is killed mid-creation the
|
|
# directory stays 0755: see Troubleshooting in the README.
|
|
- name: dsh-home
|
|
mountPath: /workspace
|
|
subPath: workspace
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: dsh-home
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|
|
# storageClassName omitted => cluster default (kind & GKE both work)
|