vm-cloudflare/observatory/alertmanager/alertmanager.yml

# Alertmanager Configuration for Cloudflare Mesh Observatory
# Phase 5B - Alerts & Escalation

global:
  # Default SMTP settings (override in receivers)
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'cloudflare-alerts@yourdomain.com'
  smtp_auth_username: '${SMTP_USERNAME}'
  smtp_auth_password: '${SMTP_PASSWORD}'
  smtp_require_tls: true

  # Slack API URL (set via environment)
  slack_api_url: '${SLACK_WEBHOOK_URL}'

  # PagerDuty integration key
  pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'

  # Resolve timeout
  resolve_timeout: 5m

# Templates for notifications
templates:
  - '/etc/alertmanager/templates/*.tmpl'

# Routing tree
route:
  # Default receiver
  receiver: 'slack-default'

  # Group alerts by these labels
  group_by: ['alertname', 'severity', 'component']

  # Wait before sending first notification
  group_wait: 30s

  # Wait before sending notification about new alerts in group
  group_interval: 5m

  # Wait before re-sending notification
  repeat_interval: 4h

  # Child routes for different severities and components
  routes:
    # ============================================
    # CRITICAL ALERTS - Immediate PagerDuty
    # ============================================
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      group_wait: 10s
      repeat_interval: 1h
      continue: true  # Also send to Slack

    - match:
        severity: critical
      receiver: 'slack-critical'
      group_wait: 10s

    # ============================================
    # TUNNEL ALERTS
    # ============================================
    - match:
        component: tunnel
      receiver: 'slack-tunnels'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'
          continue: true
        - match:
            severity: critical
          receiver: 'slack-critical'

    # ============================================
    # DNS ALERTS
    # ============================================
    - match:
        component: dns
      receiver: 'slack-dns'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'
          continue: true
        - match:
            alertname: DNSHijackDetected
          receiver: 'pagerduty-critical'

    # ============================================
    # WAF ALERTS
    # ============================================
    - match:
        component: waf
      receiver: 'slack-waf'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'
          continue: true
        - match:
            alertname: WAFMassiveAttack
          receiver: 'pagerduty-critical'

    # ============================================
    # INVARIANT ALERTS (Security Policy Violations)
    # ============================================
    - match:
        component: invariant
      receiver: 'slack-security'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'
          continue: true

    # ============================================
    # PROOFCHAIN ALERTS
    # ============================================
    - match:
        component: proofchain
      receiver: 'slack-proofchain'
      routes:
        - match:
            severity: critical
          receiver: 'pagerduty-critical'

    # ============================================
    # WARNING ALERTS - Slack only
    # ============================================
    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 8h

    # ============================================
    # INFO ALERTS - Daily digest
    # ============================================
    - match:
        severity: info
      receiver: 'email-daily'
      group_wait: 1h
      repeat_interval: 24h

    # ============================================
    # PHASE 6 - GITOPS DRIFT REMEDIATION
    # Route drift alerts to GitOps webhook for auto-PR
    # ============================================
    - match:
        alertname: DNSDriftDetected
      receiver: 'gitops-drift-pr'
      continue: true  # Also send to slack-dns

    - match:
        alertname: WAFRuleMissing
      receiver: 'gitops-drift-pr'
      continue: true

    - match:
        alertname: FirewallRuleMissing
      receiver: 'gitops-drift-pr'
      continue: true

    - match:
        alertname: TunnelConfigChanged
      receiver: 'gitops-drift-pr'
      continue: true

    - match_re:
        alertname: '.*(Drift|Mismatch|Changed).*'
      receiver: 'gitops-drift-pr'
      continue: true

# Inhibition rules - suppress lower severity when higher fires
inhibit_rules:
  # If critical fires, suppress warning for same alert
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'component']

  # If warning fires, suppress info for same alert
  - source_match:
      severity: 'warning'
    target_match:
      severity: 'info'
    equal: ['alertname', 'component']

  # Suppress all tunnel alerts if Cloudflare API is down
  - source_match:
      alertname: 'CloudflareAPIDown'
    target_match:
      component: 'tunnel'
    equal: []

  # Suppress DNS alerts during planned maintenance
  - source_match:
      alertname: 'PlannedMaintenance'
    target_match:
      component: 'dns'
    equal: []

# Receivers definition
receivers:
  # ============================================
  # SLACK RECEIVERS
  # ============================================
  - name: 'slack-default'
    slack_configs:
      - channel: '#cloudflare-alerts'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'Runbook'
            url: '{{ template "slack.cloudflare.runbook" . }}'
          - type: button
            text: 'Grafana'
            url: 'http://localhost:3000/d/cloudflare-overview'

  - name: 'slack-critical'
    slack_configs:
      - channel: '#cloudflare-critical'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: 'danger'
        actions:
          - type: button
            text: 'Runbook'
            url: '{{ template "slack.cloudflare.runbook" . }}'
          - type: button
            text: 'Grafana'
            url: 'http://localhost:3000/d/cloudflare-overview'

  - name: 'slack-warnings'
    slack_configs:
      - channel: '#cloudflare-alerts'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: 'warning'

  - name: 'slack-tunnels'
    slack_configs:
      - channel: '#cloudflare-tunnels'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'Tunnel Playbook'
            url: 'https://wiki.internal/playbooks/tunnel-rotation'
          - type: button
            text: 'Tunnel Dashboard'
            url: 'http://localhost:3000/d/tunnel-status'

  - name: 'slack-dns'
    slack_configs:
      - channel: '#cloudflare-dns'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'DNS Playbook'
            url: 'https://wiki.internal/playbooks/dns-compromise'
          - type: button
            text: 'DNS Dashboard'
            url: 'http://localhost:3000/d/dns-health'

  - name: 'slack-waf'
    slack_configs:
      - channel: '#cloudflare-waf'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'WAF Playbook'
            url: 'https://wiki.internal/playbooks/waf-incident'
          - type: button
            text: 'WAF Dashboard'
            url: 'http://localhost:3000/d/security-settings'

  - name: 'slack-security'
    slack_configs:
      - channel: '#cloudflare-security'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'Invariants Dashboard'
            url: 'http://localhost:3000/d/invariants'

  - name: 'slack-proofchain'
    slack_configs:
      - channel: '#cloudflare-proofchain'
        send_resolved: true
        title: '{{ template "slack.cloudflare.title" . }}'
        text: '{{ template "slack.cloudflare.text" . }}'
        color: '{{ template "slack.cloudflare.color" . }}'
        actions:
          - type: button
            text: 'Proofchain Dashboard'
            url: 'http://localhost:3000/d/proofchain'

  # ============================================
  # PAGERDUTY RECEIVERS
  # ============================================
  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: '${PAGERDUTY_SERVICE_KEY}'
        send_resolved: true
        description: '{{ template "pagerduty.cloudflare.description" . }}'
        severity: 'critical'
        client: 'Cloudflare Mesh Observatory'
        client_url: 'http://localhost:3000'
        details:
          alertname: '{{ .GroupLabels.alertname }}'
          component: '{{ .GroupLabels.component }}'
          severity: '{{ .GroupLabels.severity }}'
          summary: '{{ .CommonAnnotations.summary }}'
          runbook: '{{ .CommonAnnotations.runbook_url }}'

  # ============================================
  # EMAIL RECEIVERS
  # ============================================
  - name: 'email-daily'
    email_configs:
      - to: 'cloudflare-team@yourdomain.com'
        send_resolved: true
        html: '{{ template "email.cloudflare.html" . }}'
        headers:
          Subject: '[Cloudflare] Daily Alert Digest - {{ .Status | toUpper }}'

  # ============================================
  # WEBHOOK RECEIVERS (for custom integrations)
  # ============================================
  - name: 'webhook-remediation'
    webhook_configs:
      - url: 'http://autonomous-remediator:8080/webhook/alert'
        send_resolved: true
        max_alerts: 10

  # ============================================
  # PHASE 6 - GITOPS WEBHOOK RECEIVER
  # ============================================
  - name: 'gitops-drift-pr'
    webhook_configs:
      - url: '${GITOPS_WEBHOOK_URL:-http://gitops-webhook:8080/webhook/alert}'
        send_resolved: false  # Only fire on new alerts, not resolved
        max_alerts: 5
        http_config:
          # Optional: Add bearer token or basic auth
          # authorization:
          #   type: Bearer
          #   credentials: '${GITOPS_WEBHOOK_TOKEN}'