ISP congestion vs CDN failure
— recovery criteria.

GET /live/event/stream.m3u8 HTTP/1.1
host: cdn.example.com
accept: application/vnd.apple.mpegurl

----- cycle 0 (T+0m, before governed fix lands) -----
# eu-west residential cohort, single-edge synthetic session
HTTP/2 200
content-type: application/vnd.apple.mpegurl
server-timing: manifest-fetch;dur=138
x-served-by: cache-fra9921
x-cdn: cdn-A/fra9921

#EXTM3U
#EXT-X-VERSION:9
#EXT-X-TARGETDURATION:6
#EXT-X-MEDIA-SEQUENCE:117204
#EXTINF:6.000,
117204.ts

# isp.peer_congestion on the affected region:    fail  (as3356-london rtt 412ms vs 38ms baseline, as5511-frankfurt 287ms vs 41ms baseline)
# isp.last_mile_rtt:                             fail  (412ms median segment-fetch RTT vs 38ms baseline — 10.8× delta)
# tcp.retransmit_rate on the affected peer sample: fail (14.2% vs 0.4% baseline — buffer pressure)
# cache.availability on the affected edge cohort: pass (X-Cache: HIT, age=0 — CDN cache is clean)
# cache.freshness on the master playlist:         pass (age=0 vs declared max-age=2 — CDN leg healthy)
# edge.geo_drift on the affected cohort:          pass (X-Served-By: cache-fra9921 matches eu-west expected; no Atlantic cross-POP)
# multicdn.winner_RTT on cdn-a / cdn-b / cdn-c:   pass (cdn-a 38ms, cdn-b 39ms, cdn-c 41ms — all three flat at baseline; no routing flip)
# origin.reachable on this source:                pass (origin shield reached in 102ms — unaffected)
# cohort.cohort_join_stall_ratio on the affected cohort:   fail (0.086 vs 0.018 baseline, 90s window)
# cohort.cohort_rebuffer_ratio:                   fail (0.054 vs 0.012 baseline — playback stalled upstream of the cache leg)
# cohort.cohort_startup_time_p95_ms:              fail (4810ms vs 1840ms baseline — TTFF spiked)
# isp.cell_load.on_affected_region:              fail (0.71 vs 0.43 baseline — uplink pressure);
# isp.affected_region_cohort_id:                 as3356 / as5511 / as1299 (eu-west residential ISPs)
# isp.last_mile_rtt_window:                      90s (rolling 90s window — last-mile profile distinctly off-baseline)

----- cycle 1 (T+~6m, after isp.peer_congestion_reprobe + prewarmed low-bitrate variant) -----
HTTP/2 200
content-type: application/vnd.apple.mpegurl
server-timing: manifest-fetch;dur=124
x-served-by: cache-fra9921
x-cdn: cdn-A/fra9921
x-variant-class: low-bitrate-prewarm-tier

#EXTM3U
#EXT-X-VERSION:9
#EXT-X-TARGETDURATION:6
#EXT-X-MEDIA-SEQUENCE:117212

# isp.last_mile_rtt:                  164ms (down from 412ms — uplink pressure decaying under the prewarm variant)
# cohort.cohort_rebuffer_ratio:       0.018 (within baseline tolerance)
# cohort.cohort_startup_time_p95_ms:  2110ms (within baseline)
# cohort.cohort_join_stall_ratio:     0.024 (within baseline)
# cache.availability:                 pass (X-Cache: HIT — cache leg still clean)
# cache.freshness:                    pass (age=0 — CDN leg stayed clean through the entire window)
# edge.geo_drift:                     pass (X-Served-By: cache-fra9921 — no Atlantic cross-POP, even at zero egress change)
# multicdn.winner_RTT:                pass (cdn-a 38ms / cdn-b 39ms / cdn-c 41ms flat — the multi-CDN health probe proves the CDN is healthy)
# governed_action_emitted:            raise_cdn_egress_tolerance_to_rebuffer_tolerance_at_low_bitrate,
#                                      surface_peering_provider_ticket_for_as3356_as5511,
#                                      triage_isp_congestion_reprobe

{
  "stream_id": "ckeuwISPvsCDNrecover5987",
  "source": "https://cdn.example.com/live/event/stream.m3u8",
  "protocol": "HLS / CMAF / cache-fra9921 / multi-CDN cdn-a|cdn-b|cdn-c",
  "checked_at": "2026-08-19T20:14:36Z",
  "ranked_hypotheses": [
    {
      "rank": 1,
      "hypothesis": "isp_congestion",
      "tag": "dominant",
      "confidence": 0.83,
      "evidence_signals": [
        "isp.peer_congestion → fail (as3356-london 412ms vs 38ms baseline / as5511-frankfurt 287ms vs 41ms baseline / as1299-stockholm 218ms vs 33ms baseline — last-mile RTT off-baseline by 8-12× across the eu-west peer sample)",
        "isp.last_mile_rtt → fail (412ms median segment-fetch RTT vs 38ms baseline — 10.8× delta — single-edge synthetic probe_latency.mp4 reads elevated)",
        "tcp.retransmit_rate → fail (14.2% vs 0.4% baseline — buffer pressure on the residential uplink)",
        "isp.cell_load.on_affected_region → fail (0.71 vs 0.43 baseline — uplink pressure on the residential AS block)",
        "cohort.cohort_join_stall_ratio → fail (0.086 vs 0.018 baseline, 90s cohort window — late-join stalled behind the uplink)",
        "cohort.cohort_rebuffer_ratio → fail (0.054 vs 0.012 baseline — playback rebuffer spiked well above the cache-side baseline)",
        "cohort.cohort_startup_time_p95_ms → fail (4810ms vs 1840ms baseline — TTFF past the 4s threshold)",
        "cache.freshness → pass (age=0 vs declared max-age=2 — CDN cache leg is healthy)",
        "cache.availability → pass (X-Cache: HIT — cache reads pass)",
        "edge.geo_drift → pass (X-Served-By: cache-fra9921 eu-west POP — no Atlantic cross-POP)",
        "multicdn.winner_RTT → pass (cdn-a 38ms / cdn-b 39ms / cdn-c 41ms — all three flat at baseline; multi-CDN health probe proves the CDN leg is healthy)",
        "origin.reachable → pass (origin shield reached in 102ms — origin leg unaffected)"
      ]
    },
    {
      "rank": 2,
      "hypothesis": "cdn_failure",
      "tag": "ruled_out_by",
      "confidence": 0.31,
      "evidence_signals": [
        "cache.freshness probe reads pass at the failed cohort, age=0 against a max-age=2 declared TTL — the master playlist is not stale-on-revalidate; the cache leg is healthy",
        "cache.availability probe reads pass on the affected edge POP — X-Cache: HIT under rebuffer load; no edge miss posture",
        "edge.geo_drift probe reads pass on the affected cohort — X-Served-By: cache-fra9921 (expected eu POP, no Atlantic cross-POP)",
        "multicdn.winner_RTT probe reads pass on cdn-a|cdn-b|cdn-c — all three CDNs flat at baseline; no CDN routing flip; the multi-CDN health probe rules the CDN leg healthy at zero egress change",
        "origin.reachable reads pass (102ms) — origin leg unaffected; the failure cannot be CDN-shaped while the multi-CDN health probe reads pass at zero egress change",
        "if a CDN-side cache miss were stateful, the affected cohort and adjacent eu-west cohorts on the same vendor would both misfire at the cache leg — they do not; the failure is last-mile-shaped, not cache-locale-shaped"
      ]
    },
    {
      "rank": 3,
      "hypothesis": "transit_or_peering_failure",
      "tag": "alternate",
      "confidence": 0.22,
      "evidence_signals": [
        "transit link probe to the regional POP reads pass; X-Served-By resolves to cache-fra9921 in eu-west with the expected RTT range; the transit-side link is not the failure lane",
        "if a regional transit failure were stateful, a regional POP failover would re-anchor the affected cohort — the affected POP has changed event since; the failure is subscriber-side, last-mile-shaped, not transit-side",
        "the ISP-side peer RTT lifted cleanly above the 412ms median across as3356/as5511/as1299 — three AS blocks at once — too wide a footprint for a single transit failure"
      ]
    }
  ],
  "governed_actions": [
    {
      "action":       "raise_cdn_egress_tolerance_to_rebuffer_tolerance_at_low_bitrate",
      "type":         "governed",
      "decision_lane": "autonomous",
      "gating":       "confidence >= 0.80 AND cohort_join_stall_ratio_active",
      "evidence":     "top-hypothesis confidence 0.83; cohort_join_stall_ratio divergence active on the affected cohort",
      "expected_effect": "rebuffer (cohort.cohort_rebuffer_ratio) clears within ~90s as the prewarmed low-bitrate variant absorbs the per-segment RTT lift; cohort_join_stall_ratio collapses below 0.05"
    },
    {
      "action":       "triage_isp_congestion_reprobe",
      "type":         "governed",
      "decision_lane": "autonomous",
      "gating":       "isp.peer_congestion fails on ≥ 3 eu-west AS blocks",
      "evidence":     "as3356 / as5511 / as1299 lift off-baseline at the affected cohort with the same rtt_ms signature",
      "expected_effect": "peer RTT re-probe on the affected cohort — confirms the failure lane is ISP-side not CDN-side before a peering-provider ticket is filed"
    },
    {
      "action":       "surface_peering_provider_ticket_for_as3356_as5511",
      "type":         "governed",
      "decision_lane": "surfaced_to_humans",
      "gating":       "asn_block_ticket_required",
      "evidence":     "as3356 / as5511 / as1299 lift off-baseline; isp.last_mile_rtt stays > 200ms across the affected cohort for 90s+",
      "expected_effect": "on-call files a peering-provider ticket for the affected AS block; CDN side NOT touched — the multi-CDN health probe proved the CDN leg is healthy at zero egress change"
    }
  ],
  "verification_window": {
    "close_out_signal": "cohort_side_with_or_without_cache_layer_pinning",
    "probes": [
      "cohort.cohort_join_stall_ratio within tolerance over the next same-length cohort window",
      "cohort.cohort_rebuffer_ratio within tolerance over the same window",
      "cohort.cohort_startup_time_p95_ms within tolerance over the same window",
      "isp.last_mile_rtt dropping below the 1.5s baseline resilience gate (NOT the cache layer returning to green)"
    ],
    "NOT_close_out_signal": [
      "cache.availability returning to X-Cache: HIT at zero egress change — passes geometrically, does not pass cohort-side",
      "cache.freshness re-anchoring to age < max-age — same caveat",
      "the multi-CDN health probe (multicdn.winner_RTT) reading green — proves the CDN leg is healthy, but does not on its own prove the cohort is delivering green playback when last-mile is degraded"
    ],
    "explicit_note": "'CDN is healthy' ≠ 'viewers are getting green playback'. Recovery is verified cohort-side on cohort_join_stall_ratio + cohort_rebuffer_ratio + last-mile RTT (NOT on the cache layer turning green again)."
  },
  "surfaced_to_humans": [
    {"owner": "on-call peer-cohort team", "task": "open a peering-provider ticket for as3356 / as5511 / as1299 with the rtt_ms signature and the 90s window profile"},
    {"owner": "viewer-platform team",       "task": "approve the prewarmed low-bitrate variant configuration before the next weekday primetime broadcast window"},
    {"owner": "reliability team",           "task": "assign the public postmortem write-up (this page) — recovery criteria on the audit step is the cohort-side + last-mile close-out signal, NOT the cache-side close-out"}
  ]
}