{
  "actions": [
    {
      "completionEvidenceRefs": [],
      "dependencies": [],
      "description": "Treat current head dbd2126e92f70044696312e43200b3082b7eafd1 as partial candidate mitigation only. Complete release blockers or explicitly scope follow-up PRs, then merge the stable production group, fail-closed environment validation, bounded retry, URL redaction, locality tests, and offset-contract documentation; release the exact artifact and perform client-like production verification.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1 day",
      "id": "ACT-0001",
      "owner": {
        "name": "Kafka File Download Service Owners",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Current candidate prevents the exact release-created group-name drift when valid committed offsets exist and narrows per-process retry amplification; it is not complete prevention.",
      "status": "in_progress",
      "title": "Complete, merge, release, and verify the #4241 remediation set",
      "verificationMethod": "Required adjacent blockers are resolved or linked to explicit approved follow-ups; PR merged; exact commit built and promoted; production group remains FILE-DOWNLOAD-group; valid offset continuity is proven; rollout has no replay surge, 429 spike, DLQ growth, credential/signed-URL leakage, or stale on-prem overlay; CI and client-like checks pass."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0003"
      ],
      "description": "Build a dry-run-first recovery plan that deduplicates business requests, honors Nearmap rate-limit headers, applies bounded concurrency/backoff, records rollback state, and never replays the raw DLQ blindly.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1-2 days",
      "id": "ACT-0002",
      "owner": {
        "name": "Aerialytic Data Recovery and Kafka Platform",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Restores failed customer work without a second provider burst or duplicate business effects.",
      "status": "accepted",
      "title": "Recover failed jobs with deduplication and provider-wide rate bounds",
      "verificationMethod": "Signed recovery manifest; deduplicated request count; provider-rate budget; dry-run approval; bounded execution receipts; customer workflow verification; residual DLQ disposition."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [],
      "description": "Correlate source messages, outcomes, request identifiers, customer/project identities, and final job states without inferring unique impact from aggregate DLQ counts.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1 day",
      "id": "ACT-0003",
      "owner": {
        "name": "Aerialytic Incident Response and Data Engineering",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Provides a defensible recovery and communications scope.",
      "status": "in_progress",
      "title": "Produce exact impacted customer and job inventory",
      "verificationMethod": "Versioned deduplicated inventory with query hashes, inclusion/exclusion rules, unresolved records, and customer/job counts."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0002",
        "ACT-0003"
      ],
      "description": "Contact affected customers, confirm each failed design request is recovered or intentionally retried, and issue an approved factual update without overstating impact or resolution.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1-2 days",
      "id": "ACT-0004",
      "owner": {
        "name": "Aerialytic Customer Success and Support",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Closes customer-impact uncertainty and restores confidence in the affected workflow.",
      "status": "accepted",
      "title": "Complete customer recovery and approved communications",
      "verificationMethod": "Approved communication log and per-customer recovery confirmation linked to the deduplicated impact inventory."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0001"
      ],
      "description": "Block production rollout when consumer-group identity changes without an approved migration plan and prove CD cancellation cannot leave a reconciled GitOps commit in an ambiguous state.",
      "dueDate": "2026-07-17",
      "estimatedEffort": "2 days",
      "id": "ACT-0005",
      "owner": {
        "name": "Aerialytic Release Engineering",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Prevents accidental offset lineage abandonment and operator-control ambiguity.",
      "status": "accepted",
      "title": "Add offset-continuity and cancellation deployment gates",
      "verificationMethod": "Negative CI fixtures for group drift; approved migration override contract; cancellation/reconcile integration test; production release evidence includes before/after group and offset checks."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0001"
      ],
      "description": "Implement the monitoring and runbook issue, including provider header telemetry, low remaining-budget warning, 429 rate, DLQ rate/depth, consumer lag/replay pressure, and unexpected group identity.",
      "dueDate": "2026-07-17",
      "estimatedEffort": "2 days",
      "id": "ACT-0006",
      "owner": {
        "name": "Aerialytic SRE and Observability",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Detects replay and provider pressure before widespread workflow failure.",
      "status": "in_progress",
      "title": "Ship 429, DLQ, replay-pressure, and group-drift alerts with runbook #4244",
      "verificationMethod": "Issue #4244 closed with alert-rule tests, synthetic threshold exercises, dashboards, paging destinations, and runbook drill evidence."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0001"
      ],
      "description": "Define and enable least-privilege retention for deleted-pod stdout and incident-relevant Kubernetes workload logs without exposing secrets or signed URL query strings.",
      "dueDate": "2026-07-22",
      "estimatedEffort": "1 week",
      "id": "ACT-0007",
      "owner": {
        "name": "Aerialytic Cloud Platform and Security",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Preserves per-pod forensic evidence while maintaining redaction requirements.",
      "status": "accepted",
      "title": "Restore production workload log retention",
      "verificationMethod": "Retention policy approved; deleted-pod log retrieval test passes; signed URL and credential redaction tests pass; access and cost controls documented."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [],
      "description": "Replace code undefined with a stable error code, correlation ID, safe support guidance, and trace linkage while avoiding secret or provider URL disclosure.",
      "dueDate": "2026-07-22",
      "estimatedEffort": "1 week",
      "id": "ACT-0008",
      "owner": {
        "name": "Aerialytic Iframe Product and Frontend",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Improves detection, support triage, and customer trust during dependency failures.",
      "status": "accepted",
      "title": "Add customer-facing error correlation and actionable design failure states",
      "verificationMethod": "Browser test reproduces a controlled provider failure and verifies a stable safe code, correlation ID, support path, and matching server trace."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0001"
      ],
      "description": "Implement a distributed token bucket or equivalent fleet-wide gate for Nearmap, pace from X-RateLimit-Limit and X-RateLimit-Remaining, delay until X-RateLimit-Reset when exhausted, trip a provider circuit breaker on sustained 429/5xx, and enforce an explicit full-download concurrency budget across replicas. Do not assume a universal 20 RPS limit.",
      "dueDate": "2026-07-17",
      "estimatedEffort": "2-4 days",
      "id": "ACT-0009",
      "owner": {
        "name": "Kafka File Download Service Owners and SRE",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Prevents five pods with partitionsConsumedConcurrently=3 from independently generating up to 15 concurrent full downloads and synchronized retry storms.",
      "status": "accepted",
      "title": "Add provider-wide header-aware rate control and circuit breaking",
      "verificationMethod": "Multi-replica load tests prove one shared budget, header-aware pause/reset behavior, bounded attempts, circuit-open behavior, and no provider overrun at 1/3/5-pod scale."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0009"
      ],
      "description": "Heartbeat or use partition-aware ownership controls during slow downloads, bound response size instead of unbounded arrayBuffer buffering, align per-attempt and total retry time with Kafka session ownership, and design idempotent or transactional completion/DLQ plus offset handling.",
      "dueDate": "2026-07-17",
      "estimatedEffort": "3-5 days",
      "id": "ACT-0010",
      "owner": {
        "name": "Kafka Platform and File Download Service Owners",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Reduces eviction/rebalance duplicates, memory pressure, and duplicate or lost state around crashes.",
      "status": "accepted",
      "title": "Harden Kafka consumer ownership, buffering, and crash consistency",
      "verificationMethod": "Tests cover responses over the size cap, 30-second-plus provider stalls, rebalances during eachMessage, crash points around output/DLQ and offset commit, and duplicate request identifiers."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [],
      "description": "Constrain Kafka-derived folder/name joins to an approved root after canonical path validation, reject traversal and absolute paths, and run the workload as a non-root identity with minimum filesystem permissions.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1-2 days",
      "id": "ACT-0011",
      "owner": {
        "name": "Kafka File Download Service Owners and Application Security",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Removes an adjacent arbitrary-path write/read risk and reduces container compromise impact.",
      "status": "accepted",
      "title": "Remove file path traversal and root-container exposure",
      "verificationMethod": "Traversal, encoded traversal, absolute-path, symlink-escape, and valid nested-path tests pass; runtime securityContext proves non-root and least-privilege mounts."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [],
      "description": "Stop logging Kafka SASL/SSL objects, assess source and retained logs for credential/private-key exposure without printing values, remove or restrict affected logs, and perform scoped credential and certificate rotation with rollback evidence.",
      "dueDate": "2026-07-16",
      "estimatedEffort": "1 day",
      "id": "ACT-0012",
      "owner": {
        "name": "Aerialytic Security and Cloud Platform",
        "type": "team"
      },
      "priority": "P0",
      "riskReduction": "Limits the adjacent credential-disclosure risk identified during audit.",
      "status": "in_progress",
      "title": "Remove credential-bearing startup logs and rotate exposed Kafka/TLS material",
      "verificationMethod": "Code/log scans show no secret-bearing object logs; exposure assessment is signed; affected credentials are scoped and rotated; old credentials fail; workloads recover without printing secret values."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0001",
        "ACT-0005",
        "ACT-0009",
        "ACT-0010",
        "ACT-0011",
        "ACT-0012"
      ],
      "description": "Require the consumer-group, retry, rate-control, offset, idempotency, secret-redaction, health, and on-prem locality/overlay tests before image publication and deployment. Validate the exact released image and fail if Next/Staging overlay patch string matches are stale or environment/storage isolation drifts.",
      "dueDate": "2026-07-17",
      "estimatedEffort": "2-3 days",
      "id": "ACT-0013",
      "owner": {
        "name": "Aerialytic Release Engineering",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Prevents tested source from diverging from the artifact or runtime overlay that reaches production and on-prem environments.",
      "status": "accepted",
      "title": "Make incident contracts hard release gates on the actual artifact",
      "verificationMethod": "A deliberately broken group, retry, secret-log, health, and stale-overlay fixture each blocks release before deployment; promoted image digest and runtime overlay hashes match the tested release evidence."
    },
    {
      "completionEvidenceRefs": [],
      "dependencies": [
        "ACT-0010"
      ],
      "description": "Replace one-time settled registry checks and non-awaited run-loop liveness with probes that fail when the Kafka consumer exits, ownership is lost, or the processing loop is no longer making safe progress.",
      "dueDate": "2026-07-18",
      "estimatedEffort": "2 days",
      "id": "ACT-0014",
      "owner": {
        "name": "Kafka File Download Service Owners",
        "type": "team"
      },
      "priority": "P1",
      "riskReduction": "Prevents green health during a failed consumer run loop.",
      "status": "accepted",
      "title": "Make health checks reflect the live consumer run loop",
      "verificationMethod": "Fault-injection tests terminate or stall the consumer and prove readiness/liveness transitions, alerting, and bounded restart behavior."
    }
  ],
  "affectedScope": {
    "assets": [
      "kafka-file-download-rmuqop-deployment",
      "FILE-DOWNLOAD-group",
      "FILE-DOWNLOAD-rmuqop-group",
      "file-download DLQ"
    ],
    "businessUnits": [
      "Aerialytic production engineering",
      "Customer success"
    ],
    "customers": [
      "Multiple iframe customers; exact deduplicated inventory pending"
    ],
    "dataClasses": [
      "operational request metadata",
      "customer project identifiers"
    ],
    "recordCounts": {
      "badReleaseDlqOutcomes": 2222,
      "estimatedHistoricalReplayOutcomes": 4495,
      "http400DlqOutcomes": 955,
      "http429DlqOutcomes": 1267,
      "preRollbackDlqOutcomes": 2152,
      "preRollbackNewSourceMessages": 80,
      "preRollbackSuccessOutcomes": 2424,
      "preRollbackTotalOutcomes": 4576
    },
    "services": [
      "Iframe design generation",
      "File download processing",
      "Nearmap imagery requests"
    ],
    "slaImpact": "Customer-facing iframe design generation failed for multiple customers. Exact customer/job count and contractual SLA impact remain unproven."
  },
  "attachments": [
    {
      "category": "public_safe_incident_package",
      "fileSizeBytes": 31082,
      "id": "ATT-0001",
      "mimeType": "application/x-gzip",
      "name": "INC-2026-07150002-public-safe-incident-package.tar.gz",
      "sha256": "77a3f07caddbc11af6fcf4e235723c3f448e71dd0c1a48d873a2004024befc45",
      "uri": "slack://C0BF16AKVQX/1784151610.481449/F0BHKPTQNB0"
    }
  ],
  "audit": {
    "auditEvents": [
      {
        "action": "Created a new incident record separate from the 2026-07-06 data-integrity incident",
        "actor": {
          "name": "Codex incident publisher",
          "type": "automation"
        },
        "id": "AUD-0001",
        "metadata": {
          "sourceThreadId": "019f380a-bf45-7ba0-989e-ae6e93a89420"
        },
        "objectId": "INC-2026-07150002",
        "objectType": "incident",
        "timestamp": "2026-07-15T21:24:55Z"
      },
      {
        "action": "Verified original screenshot SHA-256 hashes and classified originals as restricted",
        "actor": {
          "name": "Codex incident publisher",
          "type": "automation"
        },
        "id": "AUD-0002",
        "metadata": {
          "publishedToPublicSite": false
        },
        "objectId": "EV-0001,EV-0002",
        "objectType": "evidence",
        "timestamp": "2026-07-15T21:24:55Z"
      },
      {
        "action": "Posted a new top-level private Slack incident record and attached the public-safe package in its thread",
        "actor": {
          "name": "Codex incident publisher",
          "type": "automation"
        },
        "id": "AUD-0003",
        "metadata": {
          "messagePermalink": "https://aerialytic.slack.com/archives/C0BF16AKVQX/p1784151610481449",
          "restrictedEvidenceIncluded": false,
          "slackFileId": "F0BHKPTQNB0",
          "uploadedPackageSha256": "77a3f07caddbc11af6fcf4e235723c3f448e71dd0c1a48d873a2004024befc45"
        },
        "objectId": "COM-0001",
        "objectType": "communication",
        "timestamp": "2026-07-15T21:40:10Z"
      },
      {
        "action": "Recorded immutable publication receipt and final verification boundary",
        "actor": {
          "name": "Codex incident publisher",
          "type": "automation"
        },
        "id": "AUD-0004",
        "metadata": {
          "infrastructureDesiredStateChanged": false,
          "publicationReceiptSha256": "3da1ca800f561356f84fbb9e9739241c724c34931be04abe6e9768f251a106d3"
        },
        "objectId": "EV-0008",
        "objectType": "evidence",
        "timestamp": "2026-07-15T21:42:00Z"
      }
    ],
    "createdBy": {
      "name": "Codex incident publisher",
      "type": "automation"
    },
    "legalHold": false,
    "version": 3
  },
  "category": {
    "family": "reliability",
    "id": "partial_outage",
    "name": "Partial Outage",
    "playbookBindings": [
      {
        "label": "Partial outage framework row",
        "url": "https://incidents.aerialytic.ai/files/examples/category-playbook-matrix.csv"
      },
      {
        "label": "Nearmap failed request guidance",
        "url": "https://help.nearmap.com/kb/articles/1297-capturing-information-for-failed-api-requests"
      },
      {
        "label": "Nearmap API rate-limit standard",
        "url": "https://developer.nearmap.com/docs/nearmap-api-standards"
      }
    ],
    "requiredExtensions": [
      "reliability"
    ],
    "tags": [
      "production",
      "iframe",
      "kafka",
      "consumer-group",
      "historical-replay",
      "nearmap",
      "rate-limit",
      "deployment-failure",
      "customer-escalation"
    ]
  },
  "closedAt": null,
  "communications": [
    {
      "approvals": [],
      "audience": "internal",
      "body": "New SEV-2 partial-outage record. Production rollback is healthy; permanent remediation and failed-job recovery remain pending.",
      "channel": "Slack #incident-reports",
      "createdAt": "2026-07-15T21:24:55Z",
      "externalRefs": [
        {
          "label": "New top-level Slack incident record",
          "url": "https://aerialytic.slack.com/archives/C0BF16AKVQX/p1784151610481449"
        }
      ],
      "id": "COM-0001",
      "sentAt": "2026-07-15T21:40:10Z",
      "status": "sent",
      "subject": "INC-2026-07150002 - Kafka replay and Nearmap throttling"
    },
    {
      "approvals": [],
      "audience": "customer",
      "body": "Draft only. Customer communication requires impact inventory and communications approval before sending.",
      "channel": "Customer success follow-up",
      "createdAt": "2026-07-15T21:24:55Z",
      "externalRefs": [],
      "id": "COM-0002",
      "sentAt": null,
      "status": "draft",
      "subject": "Aerialytic design-generation incident update - July 15, 2026"
    }
  ],
  "createdAt": "2026-07-15T21:24:55Z",
  "customFields": {
    "aerialytic.adjacentReleaseBlockers": [
      "Provider-wide distributed rate gate",
      "X-RateLimit header-aware pacing and reset delay",
      "Provider circuit breaker",
      "Fleet concurrency budget",
      "Consumer heartbeat and ownership-safe slow-request handling",
      "Bounded response buffering",
      "Offset continuity preflight for missing, expired, deleted, or out-of-range commits",
      "Request-level deduplication or idempotency and crash-consistent output/offset handling",
      "Path traversal removal and non-root runtime",
      "Kafka SASL/SSL credential-log removal, exposure assessment, and scoped rotation",
      "Run-loop-aware health checks",
      "Actual-artifact CI/CD enforcement",
      "Next/Staging overlay and storage-locality verification"
    ],
    "aerialytic.ciRun": "https://github.com/Aerialytic/Monorepo/actions/runs/29450818576",
    "aerialytic.currentStateLabel": "Monitoring / Remediation Pending",
    "aerialytic.forensicBoundary": "Kafka inspection was read-only and parsed only x-topic/x-error headers. Payloads and signed URLs were not displayed or persisted.",
    "aerialytic.postmortemLuck": [
      "Rollback restored a still-valid original group offset lineage.",
      "No source-data corruption or unauthorized access was evidenced.",
      "The provider throttle subsided after rollback and queue drain without a broader portal outage."
    ],
    "aerialytic.postmortemWentPoorly": [
      "Consumer-group identity changed without an offset-continuity gate.",
      "CD cancellation occurred after the GitOps commit and did not prevent reconciliation.",
      "Provider traffic had no shared rate budget or circuit breaker across replicas.",
      "The customer saw code undefined instead of a correlation ID.",
      "Deleted-pod logs were unavailable, and adjacent credential logging and path traversal risks were found during audit."
    ],
    "aerialytic.postmortemWentWell": [
      "Production was rolled back to the original image and offset lineage.",
      "The investigator kept Kafka inspection read-only and did not persist payloads or signed URLs.",
      "The failure mechanics and bad-release outcome counts were reconstructed with precise timing.",
      "The current production deployment returned to 3/3 Ready with a clean 15-minute log sample."
    ],
    "aerialytic.remediationIssue": "https://github.com/Aerialytic/Monorepo/issues/4244",
    "aerialytic.remediationPr": "https://github.com/Aerialytic/Monorepo/pull/4241",
    "aerialytic.remediationReadiness": "Partial candidate mitigation only; not live and not complete prevention.",
    "aerialytic.severityBasis": "The framework category matrix maps partial outage and proposal-generation failure for a customer segment to SEV2.",
    "aerialytic.slackIncidentRecord": "https://aerialytic.slack.com/archives/C0BF16AKVQX/p1784151610481449",
    "aerialytic.unproven": [
      "Exact per-pod record attribution",
      "Incidental at-least-once duplicates",
      "Unique business request IDs",
      "Deduplicated customer and job impact",
      "Automatic recovery of failed jobs",
      "Live provider behavior under PR #4241"
    ],
    "aerialytic.verificationComment": "https://github.com/Aerialytic/Monorepo/pull/4241#issuecomment-4985290774"
  },
  "decisions": [
    {
      "alternatives": [
        "Continue catch-up on the new group",
        "Pause consumers without rollback"
      ],
      "decision": "Roll production back to kafka-file-download 2.0.0.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "DEC-0001",
      "maker": {
        "name": "Aerialytic Release Operations",
        "type": "team"
      },
      "rationale": "Restore the original consumer-group offset lineage and stop the new-group replay class.",
      "timestamp": "2026-07-15T20:00:06Z"
    },
    {
      "alternatives": [
        "Mark resolved after rollback",
        "Close after the clean 15-minute sample"
      ],
      "decision": "Keep the incident in Monitoring / Remediation Pending.",
      "evidenceRefs": [
        "EV-0004",
        "EV-0005",
        "EV-0006"
      ],
      "id": "DEC-0002",
      "maker": {
        "name": "Aerialytic Incident Response",
        "type": "team"
      },
      "rationale": "Current runtime is healthy, but PR #4241 is not yet merged/released, exact impacted jobs are not enumerated, and failed jobs have not been recovered.",
      "timestamp": "2026-07-15T21:24:55Z"
    },
    {
      "alternatives": [
        "Publish originals",
        "Omit screenshots entirely"
      ],
      "decision": "Keep original customer screenshots restricted and publish only hashes, provenance, and redacted descriptions.",
      "evidenceRefs": [
        "EV-0001",
        "EV-0002"
      ],
      "id": "DEC-0003",
      "maker": {
        "name": "Aerialytic Security and Reliability",
        "type": "team"
      },
      "rationale": "The originals contain customer names and a project address; they are operational evidence but must not be exposed by the web app package.",
      "timestamp": "2026-07-15T21:24:55Z"
    }
  ],
  "detectedAt": "2026-07-15T19:46:51.403Z",
  "environment": {
    "applications": [
      "iframe customer portal",
      "Aerialytic design generation"
    ],
    "cloudProviders": [
      "Google Cloud"
    ],
    "clusters": [
      "production main Kubernetes cluster"
    ],
    "dependencies": [
      "Nearmap imagery provider",
      "GitOps deployment pipeline"
    ],
    "name": "Aerialytic production main",
    "regions": [],
    "services": [
      "kafka-file-download",
      "Kafka",
      "Nearmap API",
      "file-download DLQ"
    ]
  },
  "evidence": [
    {
      "category": "screenshot",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T20:00:39Z",
      "collector": {
        "name": "Restricted evidence custodian",
        "type": "system"
      },
      "confidentiality": "restricted",
      "description": "Original customer-support-channel screenshot captured at 4:00:39 PM EDT. It shows a 3:58 PM report that multiple designs were failing and includes customer names; retained only in restricted evidence and not published to the web app.",
      "fileSizeBytes": 68499,
      "id": "EV-0001",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "image/png",
      "observedAt": "2026-07-15T19:58:00Z",
      "relatedSystems": [
        "Customer support channel",
        "Iframe design generation"
      ],
      "relatedTimelineEvents": [
        "TL-0009"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": "566796aaf90ea9553d471bb166e67e940d0aa9f149667e0190ab109d6175bd33",
      "source": "Shared Mac screenshot capture",
      "tags": [
        "customer-report",
        "restricted",
        "screenshot"
      ],
      "timezone": "America/Toronto",
      "uri": null,
      "version": 1
    },
    {
      "category": "screenshot",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T20:00:48Z",
      "collector": {
        "name": "Restricted evidence custodian",
        "type": "system"
      },
      "confidentiality": "restricted",
      "description": "Original iframe workflow screenshot showing the visible 'Aerialytic error (code undefined)' symptom. The image includes a customer project address and is retained only in restricted evidence, not published to the web app.",
      "fileSizeBytes": 87240,
      "id": "EV-0002",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "image/png",
      "observedAt": "2026-07-15T20:00:48Z",
      "relatedSystems": [
        "Iframe design generation"
      ],
      "relatedTimelineEvents": [
        "TL-0009"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": "edd35bf88c3426ae1f0bd10c945be3886e7bd2e2078b691ca505945131674843",
      "source": "Shared Mac clipboard capture",
      "tags": [
        "customer-symptom",
        "restricted",
        "screenshot"
      ],
      "timezone": "America/Toronto",
      "uri": null,
      "version": 1
    },
    {
      "category": "forensic_analysis",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:24:55Z",
      "collector": {
        "name": "Restricted evidence custodian",
        "type": "system"
      },
      "confidentiality": "confidential",
      "description": "Read-only Git, Kubernetes, Kafka-header, DLQ, and log analysis establishing rollout timing, consumer-group replay mechanics, outcome counts, and rollback behavior. Kafka payloads and signed URLs were not displayed or persisted.",
      "fileSizeBytes": null,
      "id": "EV-0003",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": null,
      "observedAt": null,
      "relatedSystems": [
        "GitHub",
        "mainenv",
        "Kubernetes",
        "Kafka",
        "Nearmap API",
        "file-download DLQ"
      ],
      "relatedTimelineEvents": [
        "TL-0001",
        "TL-0002",
        "TL-0003",
        "TL-0004",
        "TL-0005",
        "TL-0006",
        "TL-0007",
        "TL-0008",
        "TL-0010",
        "TL-0011",
        "TL-0012"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": null,
      "source": "Aerialytic production incident investigation task",
      "tags": [
        "read-only",
        "forensics",
        "headers-only",
        "redacted"
      ],
      "timezone": "UTC",
      "uri": null,
      "version": 1
    },
    {
      "category": "source_change",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:24:55Z",
      "collector": {
        "name": "GitHub",
        "type": "system"
      },
      "confidentiality": "internal",
      "description": "Partial candidate mitigation PR #4241 at head dbd2126e92f70044696312e43200b3082b7eafd1. It preserves the established production group and improves retry selection/backoff, environment validation, URL redaction, locality tests, and offset documentation. It is not live and does not yet provide distributed provider pacing, header-aware reset behavior, a circuit breaker, offset safety when commits are absent, request idempotency, consumer heartbeat safety, or the adjacent security and runtime hardening tracked by this incident.",
      "fileSizeBytes": null,
      "id": "EV-0004",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "text/html",
      "observedAt": "2026-07-15T21:07:43Z",
      "relatedSystems": [
        "Aerialytic Monorepo",
        "kafka-file-download"
      ],
      "relatedTimelineEvents": [
        "TL-0013"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": null,
      "source": "https://github.com/Aerialytic/Monorepo/pull/4241",
      "tags": [
        "remediation",
        "draft-pr"
      ],
      "timezone": "UTC",
      "uri": "https://github.com/Aerialytic/Monorepo/pull/4241",
      "version": 1
    },
    {
      "category": "ci_run",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:23:23Z",
      "collector": {
        "name": "GitHub Actions",
        "type": "system"
      },
      "confidentiality": "internal",
      "description": "GitHub Actions run 29450818576 for PR #4241 head dbd2126e92f70044696312e43200b3082b7eafd1 was queued on the self-hosted runner at publication time; no green conclusion is claimed.",
      "fileSizeBytes": null,
      "id": "EV-0005",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "application/json",
      "observedAt": "2026-07-15T21:07:43Z",
      "relatedSystems": [
        "GitHub Actions",
        "self-hosted runner"
      ],
      "relatedTimelineEvents": [
        "TL-0013"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": null,
      "source": "https://github.com/Aerialytic/Monorepo/actions/runs/29450818576",
      "tags": [
        "ci",
        "queued",
        "not-green"
      ],
      "timezone": "UTC",
      "uri": "https://github.com/Aerialytic/Monorepo/actions/runs/29450818576",
      "version": 1
    },
    {
      "category": "runtime_health",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:24:55Z",
      "collector": {
        "name": "Codex incident investigator",
        "type": "automation"
      },
      "confidentiality": "internal",
      "description": "Latest read-only production health observation: main/kafka-file-download-rmuqop-deployment 3/3 Ready with zero restarts; 15-minute sample had 3 completed downloads, 0 fresh 429, 0 DLQ sends, and 0 processing errors.",
      "fileSizeBytes": null,
      "id": "EV-0006",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": null,
      "observedAt": "2026-07-15T21:24:55Z",
      "relatedSystems": [
        "main/kafka-file-download-rmuqop-deployment"
      ],
      "relatedTimelineEvents": [
        "TL-0012"
      ],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": null,
      "source": "Aerialytic production read-only checks",
      "tags": [
        "read-only",
        "healthy-runtime",
        "monitoring"
      ],
      "timezone": "UTC",
      "uri": null,
      "version": 1
    },
    {
      "category": "vendor_guidance",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:24:55Z",
      "collector": {
        "name": "Nearmap",
        "type": "vendor"
      },
      "confidentiality": "public",
      "description": "Nearmap guidance recommends exponential backoff for HTTP 429 and 5xx, not other 4xx, and documents X-RateLimit-Limit, X-RateLimit-Remaining, and X-RateLimit-Reset response headers.",
      "fileSizeBytes": null,
      "id": "EV-0007",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "text/html",
      "observedAt": "2026-07-15T21:24:55Z",
      "relatedSystems": [
        "Nearmap API"
      ],
      "relatedTimelineEvents": [],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reference-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": null,
      "source": "Nearmap Help Center and API Standard",
      "tags": [
        "vendor",
        "rate-limit",
        "retry"
      ],
      "timezone": "UTC",
      "uri": "https://developer.nearmap.com/docs/nearmap-api-standards",
      "version": 1
    },
    {
      "category": "publication_receipt",
      "chainOfCustody": [],
      "collectedAt": "2026-07-15T21:42:00Z",
      "collector": {
        "name": "Restricted evidence custodian",
        "type": "system"
      },
      "confidentiality": "confidential",
      "description": "Immutable publication receipt recording source paths, two rollback backups and hashes, restricted-evidence boundaries, Slack message/file evidence, package hashes, public and forced-VIP checks, browser proof, and the unchanged infrastructure boundary.",
      "fileSizeBytes": 4232,
      "id": "EV-0008",
      "immutable": true,
      "integrityStatus": "verified",
      "mimeType": "text/markdown",
      "observedAt": "2026-07-15T21:42:00Z",
      "relatedSystems": [
        "Incident framework",
        "edge-nginx",
        "Slack #incident-reports"
      ],
      "relatedTimelineEvents": [],
      "relatedUsers": [],
      "retentionPolicy": {
        "legalHold": false,
        "policyId": "reliability-incident-2y",
        "retainUntil": "2028-07-15"
      },
      "sha256": "3da1ca800f561356f84fbb9e9739241c724c34931be04abe6e9768f251a106d3",
      "source": "Canonical incident publication workflow",
      "tags": [
        "publication",
        "rollback",
        "checksums",
        "slack",
        "tls",
        "vip"
      ],
      "timezone": "UTC",
      "uri": null,
      "version": 1
    }
  ],
  "id": "INC-2026-07150002",
  "impactAssessments": [
    {
      "affectedRecords": null,
      "category": "availability",
      "confidence": "confirmed",
      "customerCount": null,
      "evidenceRefs": [
        "EV-0001",
        "EV-0002",
        "EV-0003"
      ],
      "metrics": {
        "badReleaseWindowEnd": "2026-07-15T20:01:56Z",
        "badReleaseWindowStart": "2026-07-15T19:38:32.823Z",
        "lastObserved429": "2026-07-15T20:36:09.138Z"
      },
      "narrative": "A core iframe design-generation workflow was unavailable or unreliable for multiple customers while other production surfaces continued operating.",
      "severity": "high"
    },
    {
      "affectedRecords": null,
      "category": "customerExperience",
      "confidence": "high",
      "customerCount": null,
      "evidenceRefs": [
        "EV-0001",
        "EV-0002"
      ],
      "metrics": {},
      "narrative": "Customers attempting to generate designs in embedded iframe workflows saw an unhelpful 'Aerialytic error (code undefined)' message. Multiple-customer impact is confirmed; exact deduplicated impact remains pending.",
      "severity": "critical"
    },
    {
      "affectedRecords": 2222,
      "category": "operational",
      "confidence": "confirmed",
      "customerCount": null,
      "evidenceRefs": [
        "EV-0003"
      ],
      "metrics": {
        "dlqOutcomes": 2222,
        "http400": 955,
        "http429": 1267
      },
      "narrative": "The new Kafka group replayed retained requests, amplified provider traffic, produced 2,222 DLQ records during the bad-release window, and requires deduplicated recovery work.",
      "severity": "high"
    },
    {
      "affectedRecords": null,
      "category": "integrity",
      "confidence": "medium",
      "customerCount": null,
      "evidenceRefs": [
        "EV-0003"
      ],
      "metrics": {},
      "narrative": "No source-data corruption is currently evidenced. Duplicate at-least-once processing outcomes occurred, and exact request-level deduplication remains unproven.",
      "severity": "low"
    },
    {
      "affectedRecords": null,
      "category": "confidentiality",
      "confidence": "medium",
      "customerCount": null,
      "evidenceRefs": [
        "EV-0003"
      ],
      "metrics": {},
      "narrative": "No evidence of unauthorized access, compromise, customer-data exposure, or exfiltration was observed. A separate audit found startup logging of Kafka SASL/SSL objects that may include credential or private-key material; exposure assessment and scoped rotation are urgent and pending.",
      "severity": "low"
    }
  ],
  "lifecycleTransitions": [
    {
      "actor": {
        "name": "Production telemetry",
        "type": "system"
      },
      "evidenceRefs": [
        "EV-0003"
      ],
      "from": "draft",
      "reason": "First observed Nearmap HTTP 429 associated with file-download processing.",
      "timestamp": "2026-07-15T19:46:51.403Z",
      "to": "open"
    },
    {
      "actor": {
        "name": "Aerialytic Incident Response",
        "type": "team"
      },
      "evidenceRefs": [
        "EV-0001",
        "EV-0002"
      ],
      "from": "open",
      "reason": "External customer report and continued iframe failures confirmed customer impact.",
      "timestamp": "2026-07-15T19:58:00Z",
      "to": "investigating"
    },
    {
      "actor": {
        "name": "Aerialytic Release Operations",
        "type": "team"
      },
      "evidenceRefs": [
        "EV-0003"
      ],
      "from": "investigating",
      "reason": "GitOps rollback restored kafka-file-download version 2.0.0.",
      "timestamp": "2026-07-15T20:00:06Z",
      "to": "mitigating"
    },
    {
      "actor": {
        "name": "Aerialytic Incident Response",
        "type": "team"
      },
      "evidenceRefs": [
        "EV-0003",
        "EV-0006"
      ],
      "from": "mitigating",
      "reason": "Last HTTP 429 in the measured window passed; production remained on the prior consumer-group lineage. Permanent remediation and failed-job recovery are still pending.",
      "timestamp": "2026-07-15T20:36:09.138Z",
      "to": "monitoring"
    }
  ],
  "ownership": {
    "approvers": [],
    "communicationsLead": {
      "name": "Aerialytic Customer Success and Support",
      "type": "team"
    },
    "engineeringLead": {
      "name": "Kafka File Download Service Owners",
      "type": "team"
    },
    "executiveSponsor": {
      "name": "Aerialytic Engineering Leadership",
      "type": "team"
    },
    "incidentCommander": {
      "name": "Aerialytic Incident Response",
      "type": "team"
    },
    "securityLead": {
      "name": "Aerialytic Security and Reliability",
      "type": "team"
    },
    "stakeholders": [
      {
        "name": "Aerialytic Release Operations",
        "type": "team"
      },
      {
        "name": "Iframe Customer Success Owners",
        "type": "team"
      }
    ]
  },
  "rca": {
    "contributingFactors": [
      {
        "actionRefs": [
          "ACT-0001"
        ],
        "description": "All replicas shared INSTANCE_ID=rmuqop, creating one new shared group that ran concurrently with the old group during rollout.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0001",
        "type": "technical"
      },
      {
        "actionRefs": [
          "ACT-0002"
        ],
        "description": "No request-level idempotency or deduplication prevented duplicate downstream provider calls.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0002",
        "type": "technical"
      },
      {
        "actionRefs": [
          "ACT-0002",
          "ACT-0006"
        ],
        "description": "Retries were process-local and no provider-wide rate limiter or circuit breaker enforced Nearmap capacity across replicas.",
        "evidenceRefs": [
          "EV-0003",
          "EV-0007"
        ],
        "id": "RCA-CF-0003",
        "type": "capacity_control"
      },
      {
        "actionRefs": [
          "ACT-0005"
        ],
        "description": "CD cancellation occurred after the GitOps image commit, creating ambiguity about whether production would still reconcile the change.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0004",
        "type": "release_control"
      },
      {
        "actionRefs": [
          "ACT-0010"
        ],
        "description": "A 30-second per-attempt timeout plus retries can exceed KafkaJS's default 30-second session timeout, while eachMessage does not heartbeat; eviction and rebalance can create duplicate processing.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0005",
        "type": "consumer_ownership"
      },
      {
        "actionRefs": [
          "ACT-0010"
        ],
        "description": "Provider responses are fully buffered with arrayBuffer and no size cap.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0006",
        "type": "resource_safety"
      },
      {
        "actionRefs": [
          "ACT-0011"
        ],
        "description": "Kafka-derived folder/name paths can traverse outside the intended root, and the container runs as root.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0007",
        "type": "security"
      },
      {
        "actionRefs": [
          "ACT-0012"
        ],
        "description": "Startup logging of Kafka SASL/SSL objects may expose credentials or private-key material in source or retained logs.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-CF-0008",
        "type": "security"
      }
    ],
    "detectionGaps": [
      {
        "actionRefs": [
          "ACT-0005"
        ],
        "description": "No deployment gate compared production consumer-group identity and committed offset continuity before rollout.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-DG-0001",
        "type": "detection"
      },
      {
        "actionRefs": [
          "ACT-0008"
        ],
        "description": "The customer UI returned code undefined instead of a traceable correlation ID and actionable failure state.",
        "evidenceRefs": [
          "EV-0002"
        ],
        "id": "RCA-DG-0002",
        "type": "customer_experience"
      },
      {
        "actionRefs": [
          "ACT-0014"
        ],
        "description": "Health checks can remain green after run-loop failure because liveness does not await the consumer Promise and the registry check is a one-time settled Promise.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-DG-0003",
        "type": "health"
      }
    ],
    "documentationGaps": [
      {
        "actionRefs": [
          "ACT-0001",
          "ACT-0005",
          "ACT-0006"
        ],
        "description": "The production offset-identity invariant and safe consumer-group migration procedure were not enforced as a documented release contract.",
        "evidenceRefs": [
          "EV-0003",
          "EV-0004"
        ],
        "id": "RCA-DOC-0001",
        "type": "runbook"
      },
      {
        "actionRefs": [
          "ACT-0013"
        ],
        "description": "Release CI does not make the incident model tests a hard dependency of release, and on-prem overlay patching can report success after string matches become stale.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-DOC-0002",
        "type": "release_contract"
      }
    ],
    "methodsUsed": [
      "Timeline correlation",
      "Change analysis",
      "Kafka offset-lineage analysis",
      "DLQ header aggregation",
      "Rolling-overlap model"
    ],
    "monitoringGaps": [
      {
        "actionRefs": [
          "ACT-0006"
        ],
        "description": "No sufficiently early alert combined group drift, replay pressure, 429 rate, and DLQ growth.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-MG-0001",
        "type": "observability"
      },
      {
        "actionRefs": [
          "ACT-0007"
        ],
        "description": "Deleted-pod stdout was unavailable because the Cloud Logging _Default sink was disabled.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-MG-0002",
        "type": "log_retention"
      }
    ],
    "rootCauses": [
      {
        "actionRefs": [
          "ACT-0001",
          "ACT-0005"
        ],
        "description": "Production consumer-group identity changed across a release, abandoning committed offsets while fromBeginning:true was enabled.",
        "evidenceRefs": [
          "EV-0003"
        ],
        "id": "RCA-ROOT-0001",
        "type": "technical"
      }
    ],
    "summary": "The rollout changed kafka-file-download from FILE-DOWNLOAD-group to FILE-DOWNLOAD-rmuqop-group. Because the new group had no committed offsets and the consumer subscribes with fromBeginning:true, the rollout replayed retained historical requests. The 66-second overlap of old and new groups amplified processing, exceeded Nearmap rate limits, grew the DLQ, and broke iframe design generation."
  },
  "regulatory": {
    "assessment": "No evidence of unauthorized access, disclosure, exfiltration, or regulated-data loss. Customer contractual communication obligations remain a customer-success and legal review item, not a breach-notification finding.",
    "deadlines": [],
    "frameworks": [],
    "jurisdictions": [],
    "possibleNotificationRequired": false
  },
  "reliability": {
    "availability": null,
    "capacity": {
      "consumerHeartbeatDuringEachMessage": false,
      "distributedRateLimiterPresent": false,
      "headerAwarePacingPresent": false,
      "mainMaximumPods": 5,
      "maximumConcurrentFullDownloadsBeforeRetries": 15,
      "partitionsConsumedConcurrentlyPerPod": 3,
      "provider": "Nearmap",
      "providerCircuitBreakerPresent": false,
      "requestDeduplicationPresent": false,
      "responseSizeCapPresent": false
    },
    "deployment": {
      "badReplicaSet": "7656f5446f",
      "badVersion": "2.0.2",
      "mainenvBadCommit": "b37d2d6",
      "mainenvRevertCommit": "16e8f8e",
      "oldNewOverlapSeconds": 66,
      "rollbackOverlapSeconds": 45,
      "stableVersion": "2.0.0"
    },
    "errorRate": {
      "badReleaseDlq": 2222,
      "http400": 955,
      "http429": 1267,
      "peak429PerMinute": 1013
    },
    "latency": {},
    "rollback": {
      "completed": true,
      "currentRuntimeHealthy": true,
      "failedJobRecoveryComplete": false,
      "postRollbackProviderPressureObserved": true,
      "restoredOriginalOffsetLineage": true,
      "restoredVersion": "2.0.0"
    },
    "slo": {
      "assessment": "Pending exact impact window and customer/job inventory"
    },
    "traffic": {
      "attributionUncertainOutcomes": 1,
      "estimatedHistoricalReplayOutcomes": 4495,
      "newSourceMessagesBeforeRollback": 80,
      "totalOutcomesBeforeRollback": 4576
    }
  },
  "resolvedAt": null,
  "security": {
    "attackVector": null,
    "compromisedAccounts": [],
    "exfiltrationAssessment": "No evidence of unauthorized access, customer-data exfiltration, or malicious activity. A separate urgent audit action is open because startup logs may contain Kafka SASL/SSL credential or private-key material; retained-log/source exposure and scoped rotation remain pending.",
    "iocs": [],
    "mitreAttack": [],
    "mitreD3fend": [],
    "notifications": [],
    "threatActor": null
  },
  "severity": {
    "confidence": "confirmed",
    "customer": "major",
    "internal": "SEV2",
    "security": "medium"
  },
  "startedAt": "2026-07-15T19:38:32.823Z",
  "status": "monitoring",
  "summary": "A production rollout changed kafka-file-download from the established FILE-DOWNLOAD-group consumer identity to FILE-DOWNLOAD-rmuqop-group while retaining fromBeginning:true. The new group had no committed offsets and replayed retained historical download requests during a 66-second old/new rolling overlap. Duplicate traffic caused Nearmap HTTP 429 throttling, file-download DLQ growth, and customer-facing iframe design-generation failures. Production was rolled back to version 2.0.0 and is currently healthy, but permanent remediation, exact impact enumeration, and recovery of failed jobs remain pending.",
  "timeline": [
    {
      "actor": {
        "name": "GitHub",
        "type": "system"
      },
      "automation": false,
      "category": "change",
      "confidence": "confirmed",
      "description": "PR #4171 merged; commit 16ab67a changed the production consumer-group identity.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0001",
      "severity": "medium",
      "source": "Git history",
      "system": "Aerialytic Monorepo",
      "timestamp": "2026-07-13T21:39:30Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "GitHub",
        "type": "system"
      },
      "automation": false,
      "category": "change",
      "confidence": "confirmed",
      "description": "PR #4205 promoted the consumer-group change into main.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0002",
      "severity": "medium",
      "source": "Git history",
      "system": "Aerialytic Monorepo",
      "timestamp": "2026-07-15T18:13:04Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Image publishing workflow",
        "type": "automation"
      },
      "automation": true,
      "category": "deployment",
      "confidence": "confirmed",
      "description": "kafka-file-download version 2.0.2 was published.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0003",
      "severity": "medium",
      "source": "Container registry and CI",
      "system": "kafka-file-download",
      "timestamp": "2026-07-15T18:57:38Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "GitOps CD",
        "type": "automation"
      },
      "automation": true,
      "category": "deployment",
      "confidence": "confirmed",
      "description": "mainenv commit b37d2d6 changed the image from 2.0.0 to 2.0.2. CD was cancelled 23 seconds later, but the GitOps commit already existed.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0004",
      "severity": "high",
      "source": "mainenv Git history",
      "system": "Production deployment",
      "timestamp": "2026-07-15T19:35:33Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Kubernetes",
        "type": "system"
      },
      "automation": true,
      "category": "deployment",
      "confidence": "confirmed",
      "description": "Production was patched to version 2.0.2 and ReplicaSet 7656f5446f appeared.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0005",
      "severity": "critical",
      "source": "Kubernetes rollout evidence",
      "system": "main/kafka-file-download-rmuqop-deployment",
      "timestamp": "2026-07-15T19:38:32.823Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Kubernetes",
        "type": "system"
      },
      "automation": true,
      "category": "failure",
      "confidence": "confirmed",
      "description": "Old and new consumer groups overlapped for 66 seconds through 19:39:47Z, creating concurrent processing from separate offset lineages.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0006",
      "severity": "critical",
      "source": "Pod lifecycle evidence",
      "system": "kafka-file-download",
      "timestamp": "2026-07-15T19:38:41Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Nearmap API",
        "type": "vendor"
      },
      "automation": true,
      "category": "detection",
      "confidence": "confirmed",
      "description": "First observed HTTP 429 response in the measured incident window.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0007",
      "severity": "critical",
      "source": "DLQ header analysis",
      "system": "Nearmap API",
      "timestamp": "2026-07-15T19:46:51.403Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "File-download DLQ",
        "type": "system"
      },
      "automation": true,
      "category": "impact",
      "confidence": "confirmed",
      "description": "The 19:47 minute peaked at 1,013 HTTP 429 DLQ records.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0008",
      "severity": "critical",
      "source": "DLQ header analysis",
      "system": "file-download DLQ",
      "timestamp": "2026-07-15T19:47:00Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "External customer support channel",
        "type": "team"
      },
      "automation": false,
      "category": "customer_impact",
      "confidence": "confirmed",
      "description": "An external customer reported 'Aerialytic error (code undefined)' while generating designs; other iframe failures continued.",
      "evidenceRefs": [
        "EV-0001",
        "EV-0002"
      ],
      "id": "TL-0009",
      "severity": "critical",
      "source": "Customer screenshot",
      "system": "Iframe design generation",
      "timestamp": "2026-07-15T19:58:00Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "GitOps CD",
        "type": "automation"
      },
      "automation": true,
      "category": "rollback",
      "confidence": "confirmed",
      "description": "mainenv revert 16e8f8e restored image version 2.0.0.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0010",
      "severity": "high",
      "source": "mainenv Git history",
      "system": "Production deployment",
      "timestamp": "2026-07-15T20:00:06Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Kubernetes",
        "type": "system"
      },
      "automation": true,
      "category": "rollback",
      "confidence": "confirmed",
      "description": "Rollback old/new overlap lasted 45 seconds; the final version 2.0.2 pod stopped at 20:01:56Z.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0011",
      "severity": "high",
      "source": "Pod lifecycle evidence",
      "system": "kafka-file-download",
      "timestamp": "2026-07-15T20:01:11Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "Nearmap API",
        "type": "vendor"
      },
      "automation": true,
      "category": "recovery",
      "confidence": "confirmed",
      "description": "Last observed HTTP 429 in the measured window. Provider throttling and queued work had continued after rollback.",
      "evidenceRefs": [
        "EV-0003"
      ],
      "id": "TL-0012",
      "severity": "medium",
      "source": "DLQ header analysis",
      "system": "Nearmap API",
      "timestamp": "2026-07-15T20:36:09.138Z",
      "timezone": "UTC",
      "visibility": "internal"
    },
    {
      "actor": {
        "name": "GitHub Actions",
        "type": "automation"
      },
      "automation": true,
      "category": "remediation",
      "confidence": "confirmed",
      "description": "Full CI for PR #4241 was queued on the self-hosted runner; it was not green at incident-publication time.",
      "evidenceRefs": [
        "EV-0005"
      ],
      "id": "TL-0013",
      "severity": "info",
      "source": "GitHub Actions run 29450818576",
      "system": "PR #4241 CI",
      "timestamp": "2026-07-15T21:07:43Z",
      "timezone": "UTC",
      "visibility": "internal"
    }
  ],
  "timezone": "America/Toronto",
  "title": "2026-07-15 Kafka file-download consumer-group replay and Nearmap throttling",
  "updatedAt": "2026-07-15T21:42:00Z"
}