# SLO + alerting for the inference platform — GPU-cluster copy.
#
# SLI:  TTFT p95 over a rolling 5m window.
# SLO:  95% of requests served with TTFT < 1s (i.e. a 5% error budget).
# The `release: kube-prometheus-stack` label makes the Prometheus Operator pick
# this rule up (matches the default ruleSelector).
#
# Mirrors ../local/manifests/slo-prometheusrule.yaml rule-for-rule — a real
# vLLM engine emits the identical `vllm:*` metric names the local mock does
# (see local/mock-vllm/app.py), so the same PromQL applies unchanged. The two
# differ in this header, the alert `service` label (`vllm-qwen` here, matching
# the real Deployment/Service name in observability.yaml — the local mock's
# rule keeps `mock-vllm`), and the `nvidia_smi_power_draw_watts` cost query
# actually resolving here, against the real nvidia_gpu_exporter (see
# diy-cluster.md) instead of reading "no data" on a GPU-less kind cluster.
# Applied by bootstrap/platform.sh's serving() layer, after observability.yaml
# (whose own SLI/SLO rule this supersedes on apply, same "inference-slo" name
# — see the note there) and synced by Argo CD via gitops/applications.yaml
# (inference-serving Application) — see the bootstrap issue for the rest of
# that layer's Service/PodMonitor wiring.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: inference-slo
  namespace: inference
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: inference.sli
      interval: 15s
      rules:
        # Recording rules — precompute the SLIs so dashboards and alerts share
        # one definition and stay cheap to evaluate.
        - record: job:vllm_ttft_p95_seconds:5m
          expr: histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))
        - record: job:vllm_ttft_p50_seconds:5m
          expr: histogram_quantile(0.50, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))
        - record: job:vllm_tokens_per_second:1m
          expr: sum(rate(vllm:generation_tokens_total[1m]))

    - name: inference.error_budget
      interval: 15s
      rules:
        # "Good" TTFT ratio at the windows a multi-window, multi-burn-rate alert
        # needs (Google SRE workbook style, collapsed to one fast pair and one
        # slow pair per the runbook's fast-burn/slow-burn alerts below). `le="1.0"`
        # is an exact bucket boundary in both the mock and real vLLM's default
        # TTFT histogram, chosen to line up with the 1s SLO threshold.
        - record: job:vllm_ttft_good_ratio:5m
          expr: sum(rate(vllm:time_to_first_token_seconds_bucket{le="1.0"}[5m])) / sum(rate(vllm:time_to_first_token_seconds_count[5m]))
        - record: job:vllm_ttft_good_ratio:30m
          expr: sum(rate(vllm:time_to_first_token_seconds_bucket{le="1.0"}[30m])) / sum(rate(vllm:time_to_first_token_seconds_count[30m]))
        - record: job:vllm_ttft_good_ratio:1h
          expr: sum(rate(vllm:time_to_first_token_seconds_bucket{le="1.0"}[1h])) / sum(rate(vllm:time_to_first_token_seconds_count[1h]))
        - record: job:vllm_ttft_good_ratio:6h
          expr: sum(rate(vllm:time_to_first_token_seconds_bucket{le="1.0"}[6h])) / sum(rate(vllm:time_to_first_token_seconds_count[6h]))

        # Burn rate = how many times faster than sustainable the 30-day error
        # budget (1 - 0.95 = 5%) is being consumed. A burn rate of 1 exhausts
        # the budget in exactly 30 days; 14.4 exhausts it in ~2 hours.
        - record: job:vllm_ttft_burn_rate:5m
          expr: (1 - job:vllm_ttft_good_ratio:5m) / (1 - 0.95)
        - record: job:vllm_ttft_burn_rate:30m
          expr: (1 - job:vllm_ttft_good_ratio:30m) / (1 - 0.95)
        - record: job:vllm_ttft_burn_rate:1h
          expr: (1 - job:vllm_ttft_good_ratio:1h) / (1 - 0.95)
        - record: job:vllm_ttft_burn_rate:6h
          expr: (1 - job:vllm_ttft_good_ratio:6h) / (1 - 0.95)

        # Availability / error-rate SLIs (runbook: 99.9% / <1%), from the
        # `finished_reason` label vLLM's own `request_success_total` counter
        # carries — "stop" is a clean completion, anything else (e.g. "abort")
        # is counted as a failed request.
        - record: job:vllm_availability_ratio:5m
          expr: sum(rate(vllm:request_success_total{finished_reason="stop"}[5m])) / sum(rate(vllm:request_success_total[5m]))
        - record: job:vllm_error_ratio:5m
          expr: 1 - job:vllm_availability_ratio:5m

    - name: inference.cost
      interval: 15s
      rules:
        # Cost per 1M output tokens, derived from GPU power draw (the
        # `nvidia_smi_power_draw_watts` gauge the nvidia_gpu_exporter exposes —
        # see diy-cluster.md) and a $/kWh rate, divided by observed throughput.
        # This promotes the footnote arithmetic in real-gpu-results.md (81 W /
        # 2,600 tok/s @ $0.17/kWh ≈ $0.0015 per 1M tokens) into a live query
        # instead of hand math. $/kWh is pinned to that same figure — swap it
        # for your local utility rate. This is the one rule in this file that
        # only resolves here: the local kind cluster has no real GPU or
        # nvidia_smi_* series, so the identical rule in
        # ../local/manifests/slo-prometheusrule.yaml reads "no data" there.
        - record: job:inference_cost_per_1m_tokens:5m
          expr: (sum(nvidia_smi_power_draw_watts) * 0.17 * 1000) / (3600 * sum(rate(vllm:generation_tokens_total[5m])))

    - name: inference.slo
      rules:
        # Primary SLO alert: TTFT p95 over budget for a sustained window.
        - alert: InferenceTTFTSLOBreach
          expr: job:vllm_ttft_p95_seconds:5m > 1
          for: 5m
          labels:
            severity: warning
            service: vllm-qwen
          annotations:
            summary: "Inference TTFT p95 over SLO (>1s)"
            description: "TTFT p95 has been {{ $value | humanizeDuration }} for 5m (SLO: <1s). The autoscaler should be adding replicas; if it isn't, check KEDA and the Prometheus triggers."

        # Saturation early-warning: queue persistently deep means TTFT is about
        # to breach even if it hasn't yet.
        - alert: InferenceQueueBacklog
          expr: sum(vllm:num_requests_waiting) > 5
          for: 2m
          labels:
            severity: warning
            service: vllm-qwen
          annotations:
            summary: "Inference request queue backing up"
            description: "{{ $value }} requests waiting for a decode slot for 2m — scale-out signal."

        # Multi-window, multi-burn-rate error-budget alerts (Google SRE
        # workbook pattern, 2-alert version): each requires agreement between
        # a long window (sustained) and a short window (currently, so it
        # clears quickly once the burn stops) before firing.
        - alert: InferenceTTFTErrorBudgetFastBurn
          expr: |
            job:vllm_ttft_burn_rate:1h > 14.4
            and
            job:vllm_ttft_burn_rate:5m > 14.4
          for: 2m
          labels:
            severity: critical
            service: vllm-qwen
          annotations:
            summary: "TTFT error budget burning fast (page)"
            description: "TTFT SLO burn rate is {{ $value | printf \"%.1f\" }}x over both the 1h and 5m windows — at this rate the 30-day error budget is gone in ~2 hours. Check replica count, KV-cache pressure, and whether KEDA is scaling out."

        - alert: InferenceTTFTErrorBudgetSlowBurn
          expr: |
            job:vllm_ttft_burn_rate:6h > 6
            and
            job:vllm_ttft_burn_rate:30m > 6
          for: 15m
          labels:
            severity: warning
            service: vllm-qwen
          annotations:
            summary: "TTFT error budget burning steadily (ticket)"
            description: "TTFT SLO burn rate is {{ $value | printf \"%.1f\" }}x over both the 6h and 30m windows — sustained low-grade degradation that will exhaust the 30-day budget in a few days if it continues. Investigate before it becomes a page."

        # Cost budget alert: catches the GPU running expensive relative to
        # throughput (saturation, thermal throttling, a stuck low-throughput
        # regime) well before anyone eyeballs a dashboard.
        - alert: InferenceCostBudgetBreach
          expr: job:inference_cost_per_1m_tokens:5m > 0.01
          for: 15m
          labels:
            severity: warning
            service: vllm-qwen
          annotations:
            summary: "Inference cost per 1M tokens over budget"
            description: "Cost/1M tokens has been ${{ $value | printf \"%.4f\" }} for 15m (budget: $0.01, observed baseline ~$0.0015-0.005 in gpu-node/real-gpu-results.md) — GPU power draw is high relative to throughput."