{ "schema_version": "0.2", "proof_id": "OCZ-PROOF-STUCK-ROLLOUT-ROLLBACK-002", "scenario_slug": "stuck-rollout-rollback", "scenario_name": "Stuck rollout / ProgressDeadlineExceeded -> rollback_k8s_deployment", "validation_status": "complete", "status_reason": "End-to-end proof run completed on 2026-05-23. All required evidence checkpoints are present: signed Prometheus webhook accepted, fail-closed evidence for unsigned rejection, OnCallZero diagnosis, rollback_k8s_deployment proposed with needs_approval=true, real Slack approval card sent and clicked via app.oncallzero.com (no tunnel), Slack signature validated (clock_skew_seconds=0), approval persisted, action_guard allowed, rollback executed, strict verification passed, incident resolved.", "proof_scope": "Owned Hetzner k3s proving cluster. Single workload: oncallzero-workloads/proving-nginx. Single scenario: ProgressDeadlineExceeded stuck rollout injected via failing readiness probe. Not customer production. Not multi-tenant. Not multi-cluster.", "public_claim_boundary": "This packet supports only the claim that an owned Hetzner k3s proving-cluster proof exists for ProgressDeadlineExceeded stuck rollout detection, Slack-approved rollback execution, and strict verification — using the stable endpoint app.oncallzero.com with signed webhook validation and no ingress tunnel. It must not be generalised to customer production, other Kubernetes environments, or all Kubernetes remediations.", "run": { "run_id": "stuck-rollout-rollback-stable-app-signed-20260523T132947Z", "incident_id": "b9abfcc7-1078-44f9-bffd-c2c54628d8a6", "incident_started_at": "2026-05-23T13:30:51.575100Z", "incident_resolved_at": "2026-05-23T13:31:45.486307Z", "total_resolution_ms": 53911, "runtime_image": "ghcr.io/runvara-io/oncallzero:sha-f753e7785b6eafaee267eb44caa3a77782cfac90", "runtime_digest": "sha256:89e178a2265cfb3ae32153520ab70da5e3d6b8106b14875f204c1a5cc5354bac", "repo_branch": "proof/server-live-slack", "repo_commit": "f753e7785b6eafaee267eb44caa3a77782cfac90" }, "environment": { "environment_type": "owned proving cluster", "cluster_type": "Hetzner k3s single-node", "cluster_node": "ubuntu-8gb-nbg1-1", "k8s_version": "v1.34.5+k3s1", "namespace": "oncallzero-workloads", "deployment": "proving-nginx", "endpoint": "https://app.oncallzero.com" }, "action_under_review": "rollback_k8s_deployment", "action_args": { "deployment_name": "proving-nginx", "namespace": "oncallzero-workloads", "name": "proving-nginx", "reason": "Rollback oncallzero-workloads/proving-nginx after failed rollout with ProgressDeadlineExceeded." }, "rollback_details": { "starting_revision": 343, "rollback_target_revision": 342, "rollback_target_rs": "proving-nginx-584f49ffbd", "rollback_target_images": [ "nginx=nginx:1.27" ], "dry_run": false }, "signed_webhook": true, "slack_approval": true, "action_guard": true, "strict_verification": true, "fail_closed_tested": true, "tunnel_provider": "", "slack_callback_via": "direct to app.oncallzero.com (no tunnel)", "evidence_present": [ { "status": "present", "kind": "metadata", "description": "Proof run metadata: proof_id, timestamp, git commit, runtime image, digest, cluster, endpoint.", "links": [ "./artifacts/metadata.json" ] }, { "status": "present", "kind": "alert_payload", "description": "Raw Prometheus alert body sent to /webhooks/prometheus with X-Webhook-Secret header.", "links": [ "./artifacts/alert-payload.json" ] }, { "status": "present", "kind": "alert_response", "description": "HTTP 202 response from /webhooks/prometheus confirming signed webhook accepted.", "links": [ "./artifacts/alert-response.json" ] }, { "status": "present", "kind": "fail_closed_evidence", "description": "Proof that unsigned webhook is rejected: no API key returns 401, correct API key + missing X-Webhook-Secret returns 403 with 'security.webhook_signature_invalid' backend log.", "links": [ "./artifacts/fail-closed-evidence.txt" ] }, { "status": "present", "kind": "incident_export", "description": "Full incident JSON export including diagnosis, attempts, approval_status=approved, execution_status=succeeded, verification_status=passed, status=resolved.", "links": [ "./artifacts/incident.json", "./artifacts/incident-awaiting-approval.json" ] }, { "status": "present", "kind": "slack_signature_validation", "description": "webhooks.slack.signature_valid log: clock_skew_seconds=0, signature_validation_reason=valid, via app.oncallzero.com (no tunnel). Slack client IP 54.210.161.182 (Slack IP range). Approver: khmuraandriy (U0AKT8RE8N9).", "links": [ "./artifacts/slack-signature-validation.txt" ] }, { "status": "present", "kind": "audit_evidence", "description": "5 audit entry IDs covering incident_opened, action_proposal, approval_request, approval_decision, and tool_call. approval_persist_complete with audit_persisted=True confirmed in logs.", "links": [ "./artifacts/audit-ids.txt" ] }, { "status": "present", "kind": "action_guard_evidence", "description": "Execution gate log sequence showing AUDIT_ACTION_EXECUTE -> rollback_k8s_deployment.start -> AUDIT_ACTION -> AUDIT_ACTION_RESULT success=True. No PERMISSION DENIED emitted. Namespace oncallzero-workloads confirmed in ONCALLZERO_ALLOWED_NAMESPACES.", "links": [ "./artifacts/action-guard-evidence.txt" ] }, { "status": "present", "kind": "verification_verdict", "description": "verification_node.verdict: passed=True, confidence=0.95. Deployment 2/2 ready, rollout complete, old stuck RS terminated. Incident status=resolved.", "links": [ "./artifacts/verification-verdict.txt" ] }, { "status": "present", "kind": "kubernetes_after_state", "description": "kubectl after-state: proving-nginx 2/2 ready, rollout successfully rolled out, active RS proving-nginx-584f49ffbd (revision 342). Two pods Running/Ready with 0 restarts.", "links": [ "./artifacts/kubectl-after.txt" ] }, { "status": "present", "kind": "rbac_evidence", "description": "kubectl auth can-i confirms oncallzero service account can patch deployments, get replicasets, list/delete pods in oncallzero-workloads namespace. RoleBinding oncallzero-workload-mutation details included.", "links": [ "./artifacts/rbac-can-i.txt" ] }, { "status": "present", "kind": "proof_flow_log", "description": "Full proof-flow log from the backend including all key log events for the incident lifecycle.", "links": [ "./artifacts/proof-flow.log" ] }, "artifacts/slack-approval-and-resolved.png" ], "evidence_redacted": [ "X-Webhook-Secret header value (referred to as in fail-closed-evidence.txt)", "X-API-Key header value (referred to as in fail-closed-evidence.txt)", "SLACK_SIGNING_SECRET value (key name referenced only in slack-signature-validation.txt)", "SLACK_BOT_TOKEN value (not present in any artifact; slack_token_source=settings.slack_bot_token in logs means it is configured, not exposed)", "SLACK_APPROVED_USER_IDS value (key name referenced only in slack-signature-validation.txt)", "Internal server filesystem paths in raw inject_output.txt and summary.txt are not published to the public site" ], "evidence_missing": [ { "status": "missing", "kind": "raw_slack_callback_payload", "description": "The raw Slack callback POST body was not separately exported. Evidence is present via backend log: webhooks.slack.callback_received with action_id=approve_action, user_id=U0AKT8RE8N9.", "links": [] }, { "status": "missing", "kind": "kubernetes_before_state_file", "description": "The before-state was captured in proof_kubectl.txt (Baseline state and Injected state sections). A standalone before-state artifact was not written separately.", "links": [ "./artifacts/proof-flow.log" ] } ], "unsupported_claims": [ "customer production remediation", "external design-partner deployment", "production readiness", "enterprise readiness", "autonomous production remediation", "proof across all Kubernetes remediations", "customer-proven or enterprise-proven", "multi-tenant isolation coverage", "all rollback scenarios or image types", "exact recovery time as a guarantee" ], "safe_claim": "On 2026-05-23, OnCallZero detected a ProgressDeadlineExceeded stuck rollout on an owned Hetzner k3s proving cluster, proposed a rollback via a signed Prometheus webhook to the stable endpoint app.oncallzero.com, sent a Slack approval card, received a real Slack Approve click with validated signature (clock_skew_seconds=0, no ingress tunnel), executed rollback_k8s_deployment, passed strict verification, and resolved the incident in approximately 54 seconds.", "unsafe_claims": [ "Do not claim this proves production-readiness or customer-grade deployment.", "Do not claim this proves all Kubernetes rollback scenarios.", "Do not claim the 54-second resolution time is a guaranteed SLA.", "Do not claim this covers multi-tenant or multi-cluster environments.", "Do not claim this was validated on a customer or design-partner cluster." ], "artifact_links": { "manifest": "./manifest.json", "metadata": "./artifacts/metadata.json", "alert_payload": "./artifacts/alert-payload.json", "alert_response": "./artifacts/alert-response.json", "fail_closed_evidence": "./artifacts/fail-closed-evidence.txt", "incident": "./artifacts/incident.json", "incident_awaiting_approval": "./artifacts/incident-awaiting-approval.json", "slack_signature_validation": "./artifacts/slack-signature-validation.txt", "audit_ids": "./artifacts/audit-ids.txt", "action_guard_evidence": "./artifacts/action-guard-evidence.txt", "verification_verdict": "./artifacts/verification-verdict.txt", "kubectl_after": "./artifacts/kubectl-after.txt", "rbac_can_i": "./artifacts/rbac-can-i.txt", "proof_flow_log": "./artifacts/proof-flow.log" } }