From c193ed21a474935cc81e9054e12f71197d747937 Mon Sep 17 00:00:00 2001 From: "SREDevOps.org Agent" Date: Wed, 21 Oct 2026 13:55:28 -0300 Subject: [PATCH] Add a startup probe so slow DSH cold starts are not killed (#6) DSH takes tens of seconds to load plugins and start serving on a small node, and every resume starts cold. The liveness budget (initialDelay 15s, period 15s, failureThreshold 3) is shorter than that, so the kubelet killed the container mid-startup. DSH exits 0 on SIGTERM, which the sandbox controller records as PodSucceeded/Finished - the pod is then never recreated and the sandbox stays unusable until a manual suspend/resume cycle. Gate liveness and readiness behind a startupProbe on /favicon.svg with a 120s budget (24 x 5s, matching the gateway's WAKE_TIMEOUT). Liveness still catches a genuinely hung server once startup succeeds. Co-authored-by: SREDevOpsAgents --- deploy/k8s/sandbox-template.yaml | 10 +++++++++- .../dsh-agents-as-a-service/10-sandboxtemplate.yaml | 13 +++++++++++++ examples/dsh-agents-as-a-service/README.md | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/deploy/k8s/sandbox-template.yaml b/deploy/k8s/sandbox-template.yaml index 3954cef..4051dfd 100644 --- a/deploy/k8s/sandbox-template.yaml +++ b/deploy/k8s/sandbox-template.yaml @@ -58,7 +58,15 @@ spec: - name: dsh-home mountPath: /home/node/.dsh # DSH_HOME # / is behind DSH's launch-token cookie (401 without it); use an - # unauthenticated path for probes. + # unauthenticated path for probes. The startup probe gives a cold + # start (plugin activation on a small node, or the first resume of a + # suspended sandbox) up to 120s before liveness can restart it. + startupProbe: + httpGet: + path: /favicon.svg + port: 3080 + periodSeconds: 5 + failureThreshold: 24 readinessProbe: httpGet: path: /favicon.svg diff --git a/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml index cebaa46..e44ac78 100644 --- a/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml +++ b/examples/dsh-agents-as-a-service/10-sandboxtemplate.yaml @@ -70,6 +70,19 @@ spec: # otherwise fail readiness and then get the pod killed by liveness. # /favicon.svg is served without auth and is the cheapest liveness # signal that proves the HTTP server is actually serving requests. + # + # Gate liveness/readiness behind a startup probe: a cold start on a + # small node takes tens of seconds to load plugins and start serving + # (a resumed sandbox starts cold too). Without it the kubelet can kill + # DSH mid-startup, and a SIGTERM'd DSH exits 0, which the sandbox + # controller records as finished. 24 x 5s = 120s, matching the + # gateway's WAKE_TIMEOUT. + startupProbe: + httpGet: + path: /favicon.svg + port: 3080 + periodSeconds: 5 + failureThreshold: 24 readinessProbe: httpGet: path: /favicon.svg diff --git a/examples/dsh-agents-as-a-service/README.md b/examples/dsh-agents-as-a-service/README.md index c24bf3e..ba0189c 100644 --- a/examples/dsh-agents-as-a-service/README.md +++ b/examples/dsh-agents-as-a-service/README.md @@ -45,7 +45,9 @@ What it demonstrates, feature by feature: - **Host allow-list**: DSH rejects unknown `Host` headers. The gateway forwards the browser's `Host`, so the public hostname must be added to `DSH_TRUSTED_HOSTS` in the template. - **Probes target `/favicon.svg`**: `/` returns `401` without the launch token, so a probe on - `/` crash-loops the pod. `/favicon.svg` is unauthenticated. + `/` crash-loops the pod. `/favicon.svg` is unauthenticated, and a `startupProbe` gates + liveness/readiness so a slow cold start (plugin activation, or a resumed sandbox) is not + restarted mid-boot. ## Prerequisites