Initial commit: Cloudflare infrastructure with WAF Intelligence

- Complete Cloudflare Terraform configuration (DNS, WAF, tunnels, access) - WAF Intelligence MCP server with threat analysis and ML classification - GitOps automation with PR workflows and drift detection - Observatory monitoring stack with Prometheus/Grafana - IDE operator rules for governed development - Security playbooks and compliance frameworks - Autonomous remediation and state reconciliation
2025-12-16 18:31:53 +00:00
commit 37a867c485
123 changed files with 25407 additions and 0 deletions
--- a/terraform/.gitlab-ci.yml
+++ b/terraform/.gitlab-ci.yml
@@ -0,0 +1,355 @@
+stages:
+  - validate
+  - plan
+  - gitops
+  - approve
+  - apply
+  - compliance
+  - reconcile
+
+variables:
+  TF_ROOT: ${CI_PROJECT_DIR}
+  TF_STATE_NAME: cloudflare-infra
+  TF_PLAN_FILE: tfplan.binary
+  TF_PLAN_JSON: tfplan.json
+
+cache:
+  key: ${CI_COMMIT_REF_SLUG}
+  paths:
+    - ${TF_ROOT}/.terraform
+
+.terraform_base:
+  image: hashicorp/terraform:1.6
+  before_script:
+    - cd ${TF_ROOT}
+    - terraform init -input=false
+
+# Stage 1: Validate
+terraform_fmt:
+  extends: .terraform_base
+  stage: validate
+  script:
+    - terraform fmt -check -recursive
+  allow_failure: false
+
+terraform_validate:
+  extends: .terraform_base
+  stage: validate
+  script:
+    - terraform validate
+  allow_failure: false
+
+# Stage 2: Plan
+terraform_plan:
+  extends: .terraform_base
+  stage: plan
+  script:
+    - terraform plan -out=${TF_PLAN_FILE} -input=false
+    - terraform show -json ${TF_PLAN_FILE} > ${TF_PLAN_JSON}
+  artifacts:
+    name: "terraform-plan-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - ${TF_ROOT}/${TF_PLAN_FILE}
+      - ${TF_ROOT}/${TF_PLAN_JSON}
+    expire_in: 7 days
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+# Stage 3: Manual Approval Gate
+manual_approval:
+  stage: approve
+  script:
+    - echo "Terraform plan approved by ${GITLAB_USER_NAME}"
+  when: manual
+  allow_failure: false
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - terraform_plan
+
+# Stage 4: Apply
+terraform_apply:
+  extends: .terraform_base
+  stage: apply
+  script:
+    - terraform apply -input=false ${TF_PLAN_FILE}
+    - terraform output -json > terraform_outputs.json
+  artifacts:
+    name: "terraform-outputs-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - ${TF_ROOT}/terraform_outputs.json
+    expire_in: 30 days
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - manual_approval
+  environment:
+    name: production
+    action: start
+
+# Stage 5: Compliance
+compliance_report:
+  stage: compliance
+  image: python:3.11-slim
+  before_script:
+    - pip install blake3
+  script:
+    - |
+      # Generate compliance snapshot
+      TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+      COMMIT_SHA=${CI_COMMIT_SHA}
+
+      # Hash all terraform files
+      find ${TF_ROOT} -name "*.tf" -exec cat {} \; | python3 -c "
+      import sys
+      import blake3
+      import json
+
+      content = sys.stdin.read()
+      tf_hash = blake3.blake3(content.encode()).hexdigest()
+
+      receipt = {
+          'receipt_type': 'terraform_compliance',
+          'schema_version': 'vm_tf_compliance_v1',
+          'timestamp': '${TIMESTAMP}',
+          'commit_sha': '${COMMIT_SHA}',
+          'pipeline_id': '${CI_PIPELINE_ID}',
+          'job_id': '${CI_JOB_ID}',
+          'tf_files_hash': tf_hash,
+          'applied_by': '${GITLAB_USER_NAME}',
+          'environment': 'production'
+      }
+
+      print(json.dumps(receipt, indent=2))
+      " > compliance_receipt.json
+
+      cat compliance_receipt.json
+  artifacts:
+    name: "compliance-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - compliance_receipt.json
+      - ${TF_ROOT}/terraform_outputs.json
+    expire_in: 365 days
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - terraform_apply
+
+# Merge Request: Plan Only
+mr_plan:
+  extends: .terraform_base
+  stage: plan
+  script:
+    - terraform plan -input=false -no-color -out=plan.tfplan | tee plan_output.txt
+    - terraform show -json plan.tfplan > plan.json
+  artifacts:
+    paths:
+      - ${TF_ROOT}/plan_output.txt
+      - ${TF_ROOT}/plan.tfplan
+      - ${TF_ROOT}/plan.json
+    expire_in: 7 days
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+# ==============================================================================
+# PHASE 6 - GITOPS PR WORKFLOWS
+# ==============================================================================
+
+# Post plan summary as MR comment
+gitops:plan_comment:
+  stage: gitops
+  image: python:3.12-slim
+  before_script:
+    - pip install requests pyyaml
+  script:
+    - |
+      cd ${CI_PROJECT_DIR}/gitops
+      python3 ci_plan_comment.py
+  variables:
+    GITLAB_TOKEN: ${GITLAB_TOKEN}
+  artifacts:
+    paths:
+      - plan_output.env
+    reports:
+      dotenv: plan_output.env
+    expire_in: 1 day
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  needs:
+    - mr_plan
+
+# Drift remediation (scheduled or alert-triggered)
+gitops:drift_remediation:
+  stage: gitops
+  image: python:3.12-slim
+  before_script:
+    - pip install requests pyyaml
+    - apt-get update && apt-get install -y git
+    - git config --global user.email "gitops-bot@cloudflare-mesh.local"
+    - git config --global user.name "GitOps Bot"
+  script:
+    - |
+      cd ${CI_PROJECT_DIR}/gitops
+      python3 drift_pr_bot.py \
+        --trigger-source "${GITOPS_TRIGGER_SOURCE:-scheduled}"
+  variables:
+    GITLAB_TOKEN: ${GITLAB_TOKEN}
+    GITOPS_DRY_RUN: "false"
+  rules:
+    # Scheduled runs
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $GITOPS_DRIFT_CHECK == "true"
+    # Alert-triggered runs
+    - if: $CI_PIPELINE_SOURCE == "trigger" && $GITOPS_TRIGGER_SOURCE == "alert"
+  needs: []
+
+# Risk gate - block high-risk changes without approval
+gitops:risk_gate:
+  stage: gitops
+  image: python:3.12-slim
+  before_script:
+    - pip install pyyaml
+  script:
+    - |
+      cd ${CI_PROJECT_DIR}/gitops
+      RISK=$(python3 plan_summarizer.py --format json | python3 -c "import sys,json; print(json.load(sys.stdin)['overall_risk'])")
+      echo "Overall risk level: $RISK"
+
+      if [ "$RISK" = "CRITICAL" ]; then
+        echo "CRITICAL risk detected. Manual approval required."
+        exit 1
+      fi
+
+      echo "Risk level acceptable for auto-merge consideration."
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  needs:
+    - mr_plan
+  allow_failure: true
+
+# Stage 6: Deep Binding - State Reconciliation
+state_reconcile:
+  stage: reconcile
+  image: python:3.11-slim
+  variables:
+    SCRIPTS_DIR: ${CI_PROJECT_DIR}/../scripts
+  before_script:
+    - pip install requests
+  script:
+    - |
+      echo "=== Cloudflare State Reconciliation ==="
+
+      # Run state reconciler
+      python3 ${CI_PROJECT_DIR}/../scripts/state-reconciler.py \
+        --zone-id ${CLOUDFLARE_ZONE_ID} \
+        --account-id ${CLOUDFLARE_ACCOUNT_ID} \
+        --output-dir ${CI_PROJECT_DIR}/../snapshots \
+        --receipt-dir ${CI_PROJECT_DIR}/../receipts
+
+      # Find latest snapshot
+      SNAPSHOT=$(ls -t ${CI_PROJECT_DIR}/../snapshots/cloudflare-*.json | head -1)
+      echo "Snapshot: $SNAPSHOT"
+
+      # Run invariant checker
+      python3 ${CI_PROJECT_DIR}/../scripts/invariant-checker.py \
+        --snapshot "$SNAPSHOT" \
+        --output-dir ${CI_PROJECT_DIR}/../anomalies || INVARIANT_FAILED=1
+
+      # Find latest report
+      REPORT=$(ls -t ${CI_PROJECT_DIR}/../anomalies/invariant-report-*.json | head -1)
+      echo "Report: $REPORT"
+
+      # Copy artifacts
+      mkdir -p reconcile_artifacts
+      cp "$SNAPSHOT" reconcile_artifacts/ 2>/dev/null || true
+      cp "$REPORT" reconcile_artifacts/ 2>/dev/null || true
+      cp ${CI_PROJECT_DIR}/../anomalies/anomaly-*.json reconcile_artifacts/ 2>/dev/null || true
+
+      # Summary
+      python3 -c "
+      import json
+      with open('$REPORT') as f:
+          r = json.load(f)
+      print(f\"Passed: {r['summary']['passed']}\")
+      print(f\"Failed: {r['summary']['failed']}\")
+      "
+
+      if [ "${INVARIANT_FAILED:-0}" = "1" ]; then
+        echo "WARNING: Invariant failures detected"
+        exit 1
+      fi
+  artifacts:
+    name: "reconcile-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - reconcile_artifacts/
+    expire_in: 365 days
+    when: always
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - compliance_report
+  allow_failure: true
+
+# Scheduled Reconciliation (Daily)
+scheduled_reconcile:
+  extends: state_reconcile
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+  needs: []
+
+# Monthly Tunnel Rotation
+monthly_rotation:
+  stage: reconcile
+  image: python:3.11-slim
+  before_script:
+    - pip install requests
+  script:
+    - |
+      echo "=== Monthly Tunnel Rotation ==="
+      python3 ${CI_PROJECT_DIR}/../scripts/tunnel-rotation-scheduler.py \
+        --account-id ${CLOUDFLARE_ACCOUNT_ID} \
+        --zone-id ${CLOUDFLARE_ZONE_ID} \
+        --max-age 90 \
+        --output-dir ${CI_PROJECT_DIR}/../receipts
+  artifacts:
+    name: "rotation-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - receipts/tunnel-rotation-*.json
+    expire_in: 365 days
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $ROTATION_CYCLE == "monthly"
+  needs: []
+
+# ProofChain Anchor (Post-Apply)
+proofchain_anchor:
+  stage: reconcile
+  image: python:3.11-slim
+  before_script:
+    - pip install requests
+  script:
+    - |
+      echo "=== ProofChain Anchoring ==="
+
+      # Run full anchor workflow
+      bash ${CI_PROJECT_DIR}/../scripts/anchor-cloudflare-state.sh \
+        --zone-id ${CLOUDFLARE_ZONE_ID} \
+        --account-id ${CLOUDFLARE_ACCOUNT_ID}
+
+      # Copy artifacts
+      mkdir -p anchor_artifacts
+      cp ${CI_PROJECT_DIR}/../snapshots/*.json anchor_artifacts/ 2>/dev/null || true
+      cp ${CI_PROJECT_DIR}/../receipts/*.json anchor_artifacts/ 2>/dev/null || true
+      cp ${CI_PROJECT_DIR}/../anomalies/*.json anchor_artifacts/ 2>/dev/null || true
+      cp ${CI_PROJECT_DIR}/../proofchain-anchors.jsonl anchor_artifacts/ 2>/dev/null || true
+
+      echo "Anchoring complete"
+  artifacts:
+    name: "anchor-${CI_COMMIT_SHORT_SHA}"
+    paths:
+      - anchor_artifacts/
+    expire_in: 365 days
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - terraform_apply
+  allow_failure: true