chore: pre-migration snapshot
Layer0, MCP servers, Terraform consolidation
This commit is contained in:
4618
.codex/Codexc
Normal file
4618
.codex/Codexc
Normal file
File diff suppressed because one or more lines are too long
25
.codex/output-guardrails.md
Normal file
25
.codex/output-guardrails.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# Output Guardrails (for Codex / agents)
|
||||
|
||||
These rules prevent config-dumps and keep agent output fast, deterministic, and reviewable.
|
||||
|
||||
## Operating rule
|
||||
|
||||
- Plan (max 3 bullets) → Execute 1 step → Report (max 5 lines).
|
||||
- Never paste full configs by default. Prefer diffs + file paths.
|
||||
- Only export full config when explicitly requested (`full=true`) and still cap output.
|
||||
- Ask for confirmation only for destructive actions.
|
||||
|
||||
## Preferred report format
|
||||
|
||||
- Goal:
|
||||
- Observed:
|
||||
- Change:
|
||||
- Command(s):
|
||||
- Result:
|
||||
- Next:
|
||||
|
||||
## Tooling rule
|
||||
|
||||
- Prefer `cf_snapshot` + `cf_config_diff` + `cf_export_config(full=false)` over any “dump” tool.
|
||||
- If output would exceed limits, write artifacts to disk and return the path.
|
||||
|
||||
@@ -27,7 +27,7 @@ export GITLAB_URL="https://gitlab.com" # Or your self-hosted URL
|
||||
# ============================================================================
|
||||
# API Token: https://dash.cloudflare.com/profile/api-tokens
|
||||
# Account ID: https://dash.cloudflare.com/ (right sidebar)
|
||||
export CLOUDFLARE_API_TOKEN="your_cloudflare_api_token_here"
|
||||
export CLOUDFLARE_API_TOKEN="nJBp4q4AxiVO29TAxwFgRYcIJYh6CY4bPPP8mW-D
|
||||
export CLOUDFLARE_ACCOUNT_ID="your_account_id_here"
|
||||
# Optional (for specific zone queries):
|
||||
export CLOUDFLARE_ZONE_ID="your_zone_id_here"
|
||||
|
||||
84
.github/workflows/registry_validation.yml
vendored
Normal file
84
.github/workflows/registry_validation.yml
vendored
Normal file
@@ -0,0 +1,84 @@
|
||||
name: Cloudflare Registry Validation
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'cloudflare/**'
|
||||
- '.github/workflows/registry_validation.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'cloudflare/**'
|
||||
- '.github/workflows/registry_validation.yml'
|
||||
schedule:
|
||||
# Daily validation to catch drift
|
||||
- cron: '0 6 * * *'
|
||||
|
||||
jobs:
|
||||
validate-registry:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Add any MCP server dependencies here if needed
|
||||
|
||||
- name: Set PYTHONPATH
|
||||
run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV
|
||||
|
||||
- name: Run Tool Name Parity Check
|
||||
run: |
|
||||
cd cloudflare
|
||||
python3 ci_check_tool_names.py
|
||||
|
||||
- name: Run Entrypoint Sanity Check
|
||||
run: |
|
||||
cd cloudflare
|
||||
python3 ci_check_entrypoints.py
|
||||
|
||||
- name: Generate Fresh Registry
|
||||
run: |
|
||||
cd cloudflare
|
||||
python3 generate_capability_registry_v2.py
|
||||
|
||||
- name: Validate Registry Format
|
||||
run: |
|
||||
cd cloudflare
|
||||
python3 -c "
|
||||
import json
|
||||
with open('capability_registry_v2.json', 'r') as f:
|
||||
registry = json.load(f)
|
||||
# Basic validation
|
||||
assert 'mcp_servers' in registry
|
||||
assert 'terraform_resources' in registry
|
||||
assert 'gitops_tools' in registry
|
||||
print('✅ Registry format is valid')
|
||||
"
|
||||
|
||||
- name: Check for Registry Changes
|
||||
id: registry_changes
|
||||
run: |
|
||||
cd cloudflare
|
||||
# Check if registry changed during validation
|
||||
if git diff --name-only capability_registry_v2.json; then
|
||||
echo "changes_detected=true" >> $GITHUB_OUTPUT
|
||||
echo "⚠️ Registry changed during validation - manual review recommended"
|
||||
else
|
||||
echo "changes_detected=false" >> $GITHUB_OUTPUT
|
||||
echo "✅ Registry is stable"
|
||||
fi
|
||||
|
||||
- name: Upload Registry Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: capability-registry
|
||||
path: cloudflare/capability_registry_v2.json
|
||||
retention-days: 30
|
||||
@@ -41,6 +41,38 @@ infra_invariants:
|
||||
- "terraform/**/*"
|
||||
- "scripts/infra-invariants.sh"
|
||||
|
||||
# ============================================================================
|
||||
# WAF + PLAN INVARIANTS
|
||||
# ============================================================================
|
||||
# Enforces WAF Intel regression + deterministic Terraform plan gating.
|
||||
|
||||
waf_plan_invariants:
|
||||
stage: validate
|
||||
image: hashicorp/terraform:latest
|
||||
before_script:
|
||||
- |
|
||||
set -euo pipefail
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache python3 py3-pip
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y python3 python3-pip
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
else
|
||||
echo "No supported package manager found to install python3/pip." >&2
|
||||
exit 1
|
||||
fi
|
||||
- python3 -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
script:
|
||||
- bash scripts/waf-and-plan-invariants.sh
|
||||
rules:
|
||||
- changes:
|
||||
- "mcp/waf_intelligence/**/*"
|
||||
- "scripts/waf-and-plan-invariants.sh"
|
||||
- "tests/test_waf_intelligence_analyzer.py"
|
||||
- "requirements-dev.txt"
|
||||
- "terraform/**/*"
|
||||
|
||||
# ============================================================================
|
||||
# PYTHON SYNTAX CHECK
|
||||
# ============================================================================
|
||||
|
||||
81
ASSURANCE.md
Normal file
81
ASSURANCE.md
Normal file
@@ -0,0 +1,81 @@
|
||||
# Assurance Run — 2025-12-18
|
||||
|
||||
- Commit: 7f2e60e1c514fbe2f459d6c2080841db7e167d85
|
||||
- Tooling: `terraform v1.5.7`, `python3 3.14.2`
|
||||
|
||||
| Check | Status | Notes |
|
||||
| --- | --- | --- |
|
||||
| `terraform fmt -recursive` | ✅ | Ran from repo root; terraform rewrote any files that diverged from canonical formatting (see `git status` for changes, if any). |
|
||||
| `terraform validate` | ⚠️ | After `terraform init`, validation succeeded but emitted deprecation warnings (`cloudflare_access_application` and `cloudflare_record.value` usage). No fixes applied. |
|
||||
| `python3 -m py_compile layer0/security_classifier.py scripts/*.py` | ✅ | All Layer0 + scripts modules compiled. |
|
||||
|
||||
Additional context:
|
||||
- `terraform init` was executed to download `cloudflare/cloudflare v4.52.5` so that validation could run; `.terraform/` and `.terraform.lock.hcl` were created/updated.
|
||||
- No other files were modified manually during this pass.
|
||||
|
||||
---
|
||||
|
||||
## Canonical Gates (CI / Audit)
|
||||
|
||||
These are the *operator-safe, auditor-grade* checks expected to pass on every sweep.
|
||||
|
||||
### 1) WAF Intel regression + CLI sanity
|
||||
|
||||
From `cloudflare/`:
|
||||
|
||||
```bash
|
||||
# Install dev deps (once)
|
||||
python3 -m pip install -r requirements-dev.txt
|
||||
|
||||
# Full test suite
|
||||
python3 -m pytest -q
|
||||
|
||||
# Analyzer regression only
|
||||
python3 -m pytest -q tests/test_waf_intelligence_analyzer.py
|
||||
|
||||
# WAF Intel CLI (must not emit false "no managed WAF" warnings)
|
||||
python3 -m mcp.waf_intelligence --file terraform/waf.tf --format json --limit 5 | python3 -m json.tool
|
||||
```
|
||||
|
||||
Acceptance:
|
||||
- Exit code 0
|
||||
- JSON parses
|
||||
- `insights` is `[]` (or informational-only; no false `"No managed WAF rules detected"` warning)
|
||||
|
||||
### 2) Terraform hardening correctness (empty-list safety + plan gates)
|
||||
|
||||
From `cloudflare/terraform/`:
|
||||
|
||||
```bash
|
||||
terraform fmt -recursive
|
||||
terraform init
|
||||
terraform validate
|
||||
|
||||
# Free-plan path (managed WAF + bot mgmt must be gated off even if flags are true)
|
||||
terraform plan -refresh=false -var-file=assurance_free.tfvars
|
||||
|
||||
# Paid-plan path (managed WAF + bot mgmt appear when flags are true)
|
||||
terraform plan -refresh=false -var-file=assurance_pro.tfvars
|
||||
```
|
||||
|
||||
Acceptance:
|
||||
- Both plans succeed (no `{}` expression errors)
|
||||
- Paid-plan run includes `cloudflare_ruleset.managed_waf` / `cloudflare_bot_management.domains`
|
||||
- Free-plan run does not include those resources
|
||||
|
||||
One-shot (runs all gates + JSON-plan assertions):
|
||||
```bash
|
||||
bash scripts/waf-and-plan-invariants.sh
|
||||
```
|
||||
|
||||
### Notes for sandboxed runs
|
||||
|
||||
Some sandboxed execution environments block Terraform provider plugins from binding unix sockets, which surfaces as:
|
||||
|
||||
```
|
||||
Unrecognized remote plugin message
|
||||
...
|
||||
listen unix ...: bind: operation not permitted
|
||||
```
|
||||
|
||||
Run Terraform with the necessary OS permissions (or outside the sandbox) in that case.
|
||||
135
CAPABILITY_REGISTRY.md
Normal file
135
CAPABILITY_REGISTRY.md
Normal file
@@ -0,0 +1,135 @@
|
||||
# Cloudflare Control Plane Capability Registry
|
||||
|
||||
Generated: 2025-12-18T02:19:38.165161+00:00
|
||||
Version: 1.0.0
|
||||
|
||||
## MCP Servers
|
||||
|
||||
### cloudflare_safe
|
||||
**Module**: `cloudflare.mcp.cloudflare_safe`
|
||||
**Purpose**: Secure Cloudflare API operations
|
||||
|
||||
**Capabilities**:
|
||||
- dns_record_management
|
||||
- waf_rule_configuration
|
||||
- tunnel_health_monitoring
|
||||
- zone_analytics_query
|
||||
- terraform_state_synchronization
|
||||
|
||||
### waf_intelligence
|
||||
**Module**: `cloudflare.mcp.waf_intelligence`
|
||||
**Purpose**: WAF rule analysis and synthesis
|
||||
|
||||
**Capabilities**:
|
||||
- waf_config_analysis
|
||||
- threat_intelligence_integration
|
||||
- compliance_mapping
|
||||
- rule_gap_identification
|
||||
- terraform_ready_rule_generation
|
||||
|
||||
### oracle_answer
|
||||
**Module**: `cloudflare.mcp.oracle_answer`
|
||||
**Purpose**: Security decision support
|
||||
|
||||
**Capabilities**:
|
||||
- security_classification
|
||||
- routing_decision_support
|
||||
- threat_assessment
|
||||
- pre_execution_screening
|
||||
|
||||
## Terraform Resources
|
||||
|
||||
### dns_management
|
||||
**Files**: dns.tf
|
||||
|
||||
**Capabilities**:
|
||||
- automated_dns_provisioning
|
||||
- spf_dmarc_mx_configuration
|
||||
- tunnel_based_routing
|
||||
- proxied_record_management
|
||||
|
||||
### waf_security
|
||||
**Files**: waf.tf
|
||||
|
||||
**Capabilities**:
|
||||
- custom_waf_rules
|
||||
- managed_ruleset_integration
|
||||
- bot_management
|
||||
- rate_limiting
|
||||
- country_blocking
|
||||
|
||||
### tunnel_infrastructure
|
||||
**Files**: tunnels.tf
|
||||
|
||||
**Capabilities**:
|
||||
- multi_service_tunnel_routing
|
||||
- ingress_rule_management
|
||||
- health_monitoring
|
||||
- credential_rotation
|
||||
|
||||
## GitOps Tools
|
||||
|
||||
### waf_rule_proposer
|
||||
**File**: gitops/waf_rule_proposer.py
|
||||
**Purpose**: Automated WAF rule generation
|
||||
|
||||
**Capabilities**:
|
||||
- threat_intel_driven_rules
|
||||
- gitlab_ci_integration
|
||||
- automated_mr_creation
|
||||
- compliance_mapping
|
||||
|
||||
### invariant_checker
|
||||
**File**: scripts/invariant_checker_py.py
|
||||
**Purpose**: Real-time state validation
|
||||
|
||||
**Capabilities**:
|
||||
- dns_integrity_checks
|
||||
- waf_compliance_validation
|
||||
- tunnel_health_monitoring
|
||||
- drift_detection
|
||||
|
||||
### drift_guardian
|
||||
**File**: scripts/drift_guardian_py.py
|
||||
**Purpose**: Automated remediation
|
||||
|
||||
**Capabilities**:
|
||||
- state_reconciliation
|
||||
- auto_remediation
|
||||
- ops_notification
|
||||
|
||||
## Security Framework
|
||||
|
||||
### layer0
|
||||
**Components**: entrypoint.py, shadow_classifier.py, preboot_logger.py
|
||||
|
||||
**Capabilities**:
|
||||
- pre_execution_security_classification
|
||||
- threat_assessment
|
||||
- security_event_logging
|
||||
- routing_decision_support
|
||||
|
||||
**Classification Levels**:
|
||||
- catastrophic
|
||||
- forbidden
|
||||
- ambiguous
|
||||
- blessed
|
||||
|
||||
## Operational Tools
|
||||
|
||||
### systemd_services
|
||||
**Services**: autonomous-remediator, drift-guardian, tunnel-rotation
|
||||
|
||||
**Capabilities**:
|
||||
- continuous_monitoring
|
||||
- automated_remediation
|
||||
- scheduled_operations
|
||||
|
||||
### test_suites
|
||||
**Test Suites**: layer0_validation, mcp_integration, cloudflare_safe_ingress
|
||||
|
||||
**Capabilities**:
|
||||
- security_classification_testing
|
||||
- mcp_server_validation
|
||||
- api_integration_testing
|
||||
|
||||
174
CAPABILITY_REGISTRY_V2.md
Normal file
174
CAPABILITY_REGISTRY_V2.md
Normal file
@@ -0,0 +1,174 @@
|
||||
# Cloudflare Control Plane Capability Registry v2
|
||||
|
||||
Generated: 2025-12-18T02:38:01.740122+00:00
|
||||
Version: 1.0.1
|
||||
|
||||
## MCP Servers
|
||||
|
||||
### cloudflare_safe
|
||||
**Module**: `cloudflare.mcp.cloudflare_safe`
|
||||
**Entrypoint**: `cloudflare.mcp.cloudflare_safe`
|
||||
**Purpose**: Secure Cloudflare API operations
|
||||
|
||||
**Tools**:
|
||||
- cf_snapshot (read/write token required)
|
||||
- cf_refresh (write token required)
|
||||
- cf_config_diff (read; requires snapshot_id)
|
||||
- cf_export_config (read)
|
||||
- cf_tunnel_status (read)
|
||||
- cf_tunnel_ingress_summary (read)
|
||||
- cf_access_policy_list (read)
|
||||
|
||||
**Auth/Env**: CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID
|
||||
**Side Effects**: read-only unless token present; cf_refresh/cf_snapshot are mutating
|
||||
**Outputs**: json, terraform_hcl
|
||||
|
||||
**Capabilities**:
|
||||
- dns_record_management
|
||||
- waf_rule_configuration
|
||||
- tunnel_health_monitoring
|
||||
- zone_analytics_query
|
||||
- terraform_state_synchronization
|
||||
|
||||
### waf_intelligence
|
||||
**Module**: `cloudflare.mcp.waf_intelligence`
|
||||
**Entrypoint**: `cloudflare.mcp.waf_intelligence.mcp_server`
|
||||
**Purpose**: WAF rule analysis and synthesis
|
||||
|
||||
**Tools**:
|
||||
- waf_capabilities (read)
|
||||
- waf_analyze (read)
|
||||
- waf_assess (read)
|
||||
- waf_generate_gitops_proposals (propose)
|
||||
|
||||
**Auth/Env**:
|
||||
**Side Effects**: propose-only; generates GitOps proposals
|
||||
**Outputs**: json, terraform_hcl, gitops_mr
|
||||
|
||||
**Capabilities**:
|
||||
- waf_config_analysis
|
||||
- threat_intelligence_integration
|
||||
- compliance_mapping
|
||||
- rule_gap_identification
|
||||
- terraform_ready_rule_generation
|
||||
|
||||
### oracle_answer
|
||||
**Module**: `cloudflare.mcp.oracle_answer`
|
||||
**Entrypoint**: `cloudflare.mcp.oracle_answer`
|
||||
**Purpose**: Security decision support
|
||||
|
||||
**Tools**:
|
||||
- oracle_answer (read)
|
||||
|
||||
**Auth/Env**:
|
||||
**Side Effects**: read-only; security classification only
|
||||
**Outputs**: json, security_classification
|
||||
|
||||
**Capabilities**:
|
||||
- security_classification
|
||||
- routing_decision_support
|
||||
- threat_assessment
|
||||
- pre_execution_screening
|
||||
|
||||
## Terraform Resources
|
||||
|
||||
### dns_management
|
||||
**Files**: dns.tf
|
||||
|
||||
**Capabilities**:
|
||||
- automated_dns_provisioning
|
||||
- spf_dmarc_mx_configuration
|
||||
- tunnel_based_routing
|
||||
- proxied_record_management
|
||||
|
||||
### waf_security
|
||||
**Files**: waf.tf
|
||||
|
||||
**Capabilities**:
|
||||
- custom_waf_rules
|
||||
- managed_ruleset_integration
|
||||
- bot_management
|
||||
- rate_limiting
|
||||
- country_blocking
|
||||
|
||||
### tunnel_infrastructure
|
||||
**Files**: tunnels.tf
|
||||
|
||||
**Capabilities**:
|
||||
- multi_service_tunnel_routing
|
||||
- ingress_rule_management
|
||||
- health_monitoring
|
||||
- credential_rotation
|
||||
|
||||
## GitOps Tools
|
||||
|
||||
### waf_rule_proposer
|
||||
**File**: gitops/waf_rule_proposer.py
|
||||
**Purpose**: Automated WAF rule generation
|
||||
**Side Effects**: creates GitLab merge requests
|
||||
**Outputs**: terraform_hcl, gitops_mr
|
||||
|
||||
**Capabilities**:
|
||||
- threat_intel_driven_rules
|
||||
- gitlab_ci_integration
|
||||
- automated_mr_creation
|
||||
- compliance_mapping
|
||||
|
||||
### invariant_checker
|
||||
**File**: scripts/invariant_checker_py.py
|
||||
**Purpose**: Real-time state validation
|
||||
**Side Effects**: generates anomaly reports
|
||||
**Outputs**: json, anomaly_report
|
||||
|
||||
**Capabilities**:
|
||||
- dns_integrity_checks
|
||||
- waf_compliance_validation
|
||||
- tunnel_health_monitoring
|
||||
- drift_detection
|
||||
|
||||
### drift_guardian
|
||||
**File**: scripts/drift_guardian_py.py
|
||||
**Purpose**: Automated remediation
|
||||
**Side Effects**: applies Terraform changes
|
||||
**Outputs**: terraform_apply, remediation_report
|
||||
|
||||
**Capabilities**:
|
||||
- state_reconciliation
|
||||
- auto_remediation
|
||||
- ops_notification
|
||||
|
||||
## Security Framework
|
||||
|
||||
### layer0
|
||||
**Components**: entrypoint.py, shadow_classifier.py, preboot_logger.py
|
||||
|
||||
**Capabilities**:
|
||||
- pre_execution_security_classification
|
||||
- threat_assessment
|
||||
- security_event_logging
|
||||
- routing_decision_support
|
||||
|
||||
**Classification Levels**:
|
||||
- catastrophic
|
||||
- forbidden
|
||||
- ambiguous
|
||||
- blessed
|
||||
|
||||
## Operational Tools
|
||||
|
||||
### systemd_services
|
||||
**Services**: autonomous-remediator, drift-guardian, tunnel-rotation
|
||||
|
||||
**Capabilities**:
|
||||
- continuous_monitoring
|
||||
- automated_remediation
|
||||
- scheduled_operations
|
||||
|
||||
### test_suites
|
||||
**Test Suites**: layer0_validation, mcp_integration, cloudflare_safe_ingress
|
||||
|
||||
**Capabilities**:
|
||||
- security_classification_testing
|
||||
- mcp_server_validation
|
||||
- api_integration_testing
|
||||
|
||||
151
CONTROL_PLANE_README.md
Normal file
151
CONTROL_PLANE_README.md
Normal file
@@ -0,0 +1,151 @@
|
||||
# Cloudflare Control Plane
|
||||
|
||||
A programmable, verifiable, policy-driven Cloudflare operating system with MCP as the primary interface layer.
|
||||
|
||||
## 🏛️ Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Intent Layer │◄──►│ State Layer │◄──►│ Verify Layer │
|
||||
│ (MCP Servers) │ │ (Terraform) │ │ (Invariants) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ GitOps Layer │◄──►│ Cloudflare API │◄──►│ Security Layer │
|
||||
│ (Automation) │ │ (Live State) │ │ (Layer0) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
## 🎯 Core Components
|
||||
|
||||
### MCP Interface Layer (Intent)
|
||||
- **`cloudflare.mcp.cloudflare_safe`**: State mutation operations
|
||||
- **`cloudflare.mcp.waf_intelligence`**: Analysis & rule synthesis
|
||||
- **`cloudflare.mcp.oracle_answer`**: Security decision support
|
||||
|
||||
### Terraform State Layer (Desired State)
|
||||
- **DNS Management**: Automated DNS, SPF/DMARC, tunnel routing
|
||||
- **WAF Security**: Custom rules + managed rulesets
|
||||
- **Tunnel Infrastructure**: Multi-service ingress routing
|
||||
- **Bot Management**: Automated detection & mitigation
|
||||
|
||||
### GitOps Automation Layer (Change Propagation)
|
||||
- **WAF Rule Proposer**: Threat-intel driven rule generation
|
||||
- **Invariant Checker**: Real-time state validation
|
||||
- **Drift Guardian**: Automated remediation
|
||||
- **CI/CD Integration**: GitHub Actions + GitLab CI
|
||||
|
||||
### Security Framework Layer (Verification)
|
||||
- **Layer0**: Pre-execution security classification
|
||||
- **Shadow Classifier**: Threat assessment
|
||||
- **Preboot Logger**: Security event tracking
|
||||
- **Invariant Validation**: Continuous compliance checking
|
||||
|
||||
## 🔄 Operational Flows
|
||||
|
||||
### Threat Intelligence → WAF Enforcement
|
||||
```
|
||||
Threat Intel → WAF Proposal → MR Review → Terraform Apply → Invariant Check → Remediation
|
||||
```
|
||||
|
||||
### DNS/Tunnel Management
|
||||
```
|
||||
Service Definition → Tunnel Config → DNS Routing → Health Monitoring → Rotation
|
||||
```
|
||||
|
||||
### Security Classification
|
||||
```
|
||||
Query → Layer0 Classification → Routing Decision → Execution/Block → Logging
|
||||
```
|
||||
|
||||
## 🛡️ Security Posture
|
||||
|
||||
### Risk Mitigations
|
||||
- **Token Scoping**: Least-privilege API tokens
|
||||
- **Rate Limiting**: API call throttling
|
||||
- **Audit Trail**: Comprehensive logging
|
||||
- **Invariant Checks**: Real-time compliance validation
|
||||
|
||||
### Compliance Frameworks
|
||||
- PCI-DSS 6.6
|
||||
- OWASP-ASVS 13
|
||||
- Zero-trust architecture
|
||||
|
||||
## 🚀 MCP Server Capabilities
|
||||
|
||||
### Cloudflare Safe MCP
|
||||
```bash
|
||||
# Tools available
|
||||
- dns_record_manage
|
||||
- waf_rule_configure
|
||||
- tunnel_health_check
|
||||
- zone_analytics_query
|
||||
```
|
||||
|
||||
### WAF Intelligence MCP
|
||||
```bash
|
||||
# Tools available
|
||||
- waf_config_analyze
|
||||
- threat_intel_integrate
|
||||
- compliance_map_generate
|
||||
- rule_gap_identify
|
||||
```
|
||||
|
||||
### Oracle Answer MCP
|
||||
```bash
|
||||
# Tools available
|
||||
- security_classify
|
||||
- routing_decide
|
||||
- threat_assess
|
||||
- decision_support
|
||||
```
|
||||
|
||||
## 📊 Monitoring & Observability
|
||||
|
||||
### Key Metrics
|
||||
- DNS resolution latency
|
||||
- WAF rule effectiveness
|
||||
- Tunnel health status
|
||||
- API rate limit utilization
|
||||
- Invariant compliance rate
|
||||
|
||||
### Alerting Triggers
|
||||
- Invariant violations
|
||||
- Tunnel connectivity issues
|
||||
- WAF rule deployment failures
|
||||
- Security classification anomalies
|
||||
|
||||
## 🔧 Development & Extension
|
||||
|
||||
### Adding New MCP Servers
|
||||
1. Follow wrapper pattern in `/.secret/mcp/template.sh`
|
||||
2. Add health checks and PYTHONPATH injection
|
||||
3. Register in OpenCode configuration
|
||||
4. Add to smoke test (`/test_mcp_servers.sh`)
|
||||
|
||||
### Extending Terraform Modules
|
||||
- Maintain compatibility with existing state
|
||||
- Add corresponding invariant checks
|
||||
- Update GitOps automation
|
||||
|
||||
### Security Framework Integration
|
||||
- Extend Layer0 classification rules
|
||||
- Add new threat intelligence sources
|
||||
- Enhance compliance mappings
|
||||
|
||||
## 🎯 Production Readiness
|
||||
|
||||
### ✅ Completed
|
||||
- Deterministic MCP interfaces
|
||||
- GitOps automation pipeline
|
||||
- Real-time invariant checking
|
||||
- Security classification framework
|
||||
|
||||
### 🔄 Operational Excellence
|
||||
- Automated remediation
|
||||
- Comprehensive monitoring
|
||||
- Audit trail preservation
|
||||
- Compliance validation
|
||||
|
||||
This control plane represents a **foundational infrastructure layer** that can support higher-level automation, agent systems, and compliance proofs without architectural changes.
|
||||
@@ -1,6 +1,7 @@
|
||||
# LAYER 0 SHADOW
|
||||
|
||||
Pre-Boot Cognition Guard | Ouroboric Gate
|
||||
Public label: Intent Safety Kernel
|
||||
Version: 1.0 (Rubedo Seal)
|
||||
Status: Active Primitive
|
||||
Implements: Nigredo -> Rubedo (pre-form cognition)
|
||||
@@ -27,6 +28,13 @@ Guarantees:
|
||||
- Ambiguous intent does not awaken the wrong agent chain.
|
||||
- Catastrophic requests are contained and recorded, not processed.
|
||||
|
||||
### 2.1 Invariant Guarantees (Immutables)
|
||||
|
||||
Layer 0 is intentionally constrained. These invariants are non-negotiable:
|
||||
- Layer 0 does not load doctrine, select agents, or invoke MCP tools.
|
||||
- Layer 0 produces no side effects beyond preboot anomaly logging for forbidden/catastrophic outcomes.
|
||||
- Telemetry-driven learning may only add/strengthen detections (escalate); it must not relax catastrophic boundaries without replay validation and explicit review.
|
||||
|
||||
---
|
||||
|
||||
## 3. Classification Model
|
||||
@@ -105,6 +113,10 @@ Notes:
|
||||
- blessed and ambiguous queries are not logged here; only violations appear.
|
||||
- catastrophic requests reveal no additional context to the requester.
|
||||
|
||||
### 6.1 Risk Score Semantics
|
||||
|
||||
`risk_score` is an ordinal signal (0-5) used for triage and audit correlation. It is monotonic under learning, may be context-weighted (e.g., production accounts), and does not decay without replay validation.
|
||||
|
||||
---
|
||||
|
||||
## 7. Interaction With Higher Layers
|
||||
|
||||
@@ -91,7 +91,7 @@ Each account becomes its own MCP entry, wired to its own env vars:
|
||||
// Production Cloudflare account
|
||||
"cloudflare_prod": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-cloudflare"],
|
||||
"command": ["python3", "-m", "mcp.cloudflare_safe"],
|
||||
"environment": {
|
||||
"CLOUDFLARE_API_TOKEN": "{env:CLOUDFLARE_API_TOKEN_PRODUCTION}",
|
||||
"CLOUDFLARE_ACCOUNT_ID": "{env:CLOUDFLARE_ACCOUNT_ID_PRODUCTION}"
|
||||
@@ -102,7 +102,7 @@ Each account becomes its own MCP entry, wired to its own env vars:
|
||||
// Staging Cloudflare account
|
||||
"cloudflare_staging": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-cloudflare"],
|
||||
"command": ["python3", "-m", "mcp.cloudflare_safe"],
|
||||
"environment": {
|
||||
"CLOUDFLARE_API_TOKEN": "{env:CLOUDFLARE_API_TOKEN_STAGING}",
|
||||
"CLOUDFLARE_ACCOUNT_ID": "{env:CLOUDFLARE_ACCOUNT_ID_STAGING}"
|
||||
@@ -253,8 +253,8 @@ Cursor IDE itself uses a single account (your Cursor subscription), but Cursor A
|
||||
}
|
||||
},
|
||||
"cloudflare_prod": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-cloudflare"],
|
||||
"command": "python3",
|
||||
"args": ["-m", "mcp.cloudflare_safe"],
|
||||
"env": {
|
||||
"CLOUDFLARE_API_TOKEN": "prod_token",
|
||||
"CLOUDFLARE_ACCOUNT_ID": "prod_account_id"
|
||||
|
||||
153
OPERATIONAL_FLOWS.md
Normal file
153
OPERATIONAL_FLOWS.md
Normal file
@@ -0,0 +1,153 @@
|
||||
# Cloudflare Control Plane Operational Flows
|
||||
|
||||
## 🔄 Threat Intelligence → WAF Enforcement Flow
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Threat Intel │───►│ WAF Intel MCP │───►│ GitOps MR │
|
||||
│ Collector │ │ (Analysis) │ │ (Proposal) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Classification │◄──►│ Rule Synthesis │◄──►│ MR Automation │
|
||||
│ (ML/Intel) │ │ (Generator) │ │ (CI/CD) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Compliance Map │───►│ Terraform Apply │───►│ Invariant Check │
|
||||
│ (Mapper) │ │ (Safe MCP) │ │ (Validator) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Audit Trail │◄───│ Live State │◄───│ Remediation │
|
||||
│ (Logger) │ │ (Cloudflare) │ │ (Guardian) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Flow Steps:
|
||||
1. **Threat Intel Collection**: Gather indicators from external sources
|
||||
2. **WAF Intelligence Analysis**: ML classification + rule gap analysis
|
||||
3. **Rule Proposal**: Generate Terraform-ready WAF rules
|
||||
4. **GitOps MR**: Automated merge request creation
|
||||
5. **Compliance Mapping**: Attach PCI-DSS/OWASP compliance data
|
||||
6. **Terraform Apply**: Safe MCP server applies changes
|
||||
7. **Invariant Validation**: Real-time state verification
|
||||
8. **Remediation**: Automated fix if invariants violated
|
||||
|
||||
## 🌐 DNS/Tunnel Management Flow
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Service Def │───►│ Tunnel Config │───►│ DNS Routing │
|
||||
│ (Manifest) │ │ (Terraform) │ │ (Records) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Health Monitor │◄──►│ Safe MCP Apply │◄──►│ Invariant Check │
|
||||
│ (Checker) │ │ (Mutation) │ │ (DNS/Tunnel) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Rotation Sched │───►│ Credential Rot │───►│ Audit Logging │
|
||||
│ (Timer) │ │ (Automation) │ │ (Compliance) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Flow Steps:
|
||||
1. **Service Definition**: Define service endpoints and requirements
|
||||
2. **Tunnel Configuration**: Create Cloudflare Tunnel ingress rules
|
||||
3. **DNS Routing**: Point domains/subdomains to tunnel endpoints
|
||||
4. **Health Monitoring**: Continuous tunnel connectivity checks
|
||||
5. **Safe MCP Operations**: Programmatic DNS/tunnel management
|
||||
6. **Invariant Validation**: DNS integrity + tunnel health checks
|
||||
7. **Credential Rotation**: Automated tunnel secret rotation
|
||||
8. **Audit Logging**: Comprehensive operational tracking
|
||||
|
||||
## 🛡️ Security Classification Flow
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ User Query │───►│ Layer0 Classify │───►│ Routing Decision │
|
||||
│ (Input) │ │ (Pre-exec) │ │ (Action) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Shadow Eval │◄──►│ Oracle Answer │◄──►│ Security Context │
|
||||
│ (Classifier) │ │ (MCP Server) │ │ (Environment) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Preboot Log │───►│ Execute/Block │───►│ Audit Trail │
|
||||
│ (Security) │ │ (Decision) │ │ (Compliance) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Flow Steps:
|
||||
1. **User Query Input**: Receive command/query from user/agent
|
||||
2. **Layer0 Classification**: Pre-execution security assessment
|
||||
3. **Routing Decision**: Determine allow/block/redirect action
|
||||
4. **Shadow Evaluation**: ML-based threat assessment
|
||||
5. **Oracle Answer**: Security decision support via MCP
|
||||
6. **Preboot Logging**: Security event recording
|
||||
7. **Execution/Block**: Allow safe operations, block dangerous ones
|
||||
8. **Audit Trail**: Comprehensive security event tracking
|
||||
|
||||
## 🔄 Continuous Verification Loop
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Live State │───►│ Invariant Check │───►│ Anomalies │
|
||||
│ (Cloudflare) │ │ (Validator) │ │ (Detection) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ Drift Detect │◄──►│ Auto Remediate │◄──►│ Notify Ops │
|
||||
│ (Guardian) │ │ (Fixer) │ │ (Alerting) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
||||
│ State Update │───►│ Re-check Inv │───►│ Close Loop │
|
||||
│ (Terraform) │ │ (Validation) │ │ (Complete) │
|
||||
└─────────────────┘ └─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Flow Steps:
|
||||
1. **Live State Monitoring**: Continuous Cloudflare API polling
|
||||
2. **Invariant Validation**: Check against desired state + security policies
|
||||
3. **Anomaly Detection**: Identify configuration drift or violations
|
||||
4. **Drift Analysis**: Determine root cause and severity
|
||||
5. **Auto Remediation**: Apply fixes via Safe MCP server
|
||||
6. **Ops Notification**: Alert human operators if needed
|
||||
7. **State Update**: Apply Terraform changes if remediation successful
|
||||
8. **Re-validation**: Confirm invariants are restored
|
||||
|
||||
## 🎯 Key Operational Principles
|
||||
|
||||
### Separation of Concerns
|
||||
- **MCP = Intent**: What should happen
|
||||
- **Terraform = State**: What the desired state is
|
||||
- **GitOps = Change**: How changes propagate
|
||||
- **Layer0 = Security**: Whether actions are safe
|
||||
|
||||
### Deterministic Operations
|
||||
- Same inputs → same outputs
|
||||
- No ambient dependencies
|
||||
- Explicit environment configuration
|
||||
- Version-controlled everything
|
||||
|
||||
### Continuous Verification
|
||||
- Real-time state validation
|
||||
- Automated remediation
|
||||
- Comprehensive audit trails
|
||||
- Security classification at every step
|
||||
|
||||
These flows represent a **production-grade operational model** where each component has clear responsibilities and the system self-corrects when deviations occur.
|
||||
@@ -251,6 +251,10 @@ class ShadowClassifier:
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `layer0_risk_score` is an ordinal signal (0-5) used for triage and audit correlation, and may be context-weighted (e.g., production accounts).
|
||||
- Telemetry-driven learning should be monotonic (escalate-only) unless replay validation explicitly approves relaxation.
|
||||
|
||||
### Key Metrics for Learning
|
||||
|
||||
1. **Classification Accuracy**
|
||||
|
||||
121
REGISTRY_ENHANCEMENT_SUMMARY.md
Normal file
121
REGISTRY_ENHANCEMENT_SUMMARY.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# Cloudflare Control Plane Registry Enhancement Summary
|
||||
|
||||
## ✅ Enhanced Capability Registry (v1.0.1)
|
||||
|
||||
**Key improvements implemented:**
|
||||
|
||||
### 1. **Exact MCP Tool Names & Entrypoints**
|
||||
- **Cloudflare Safe**: `cf_snapshot`, `cf_refresh`, `cf_config_diff`, etc.
|
||||
- **WAF Intelligence**: `waf_capabilities`, `waf_analyze`, `waf_assess`, etc.
|
||||
- **Oracle Answer**: `oracle_answer`
|
||||
- **Entrypoints**: Exact Python module paths for execution
|
||||
|
||||
### 2. **Operational Metadata**
|
||||
- **Auth/Env**: Required environment variables per server
|
||||
- **Side Effects**: Clear indication of read-only vs. mutating operations
|
||||
- **Outputs**: Specific output formats (JSON, Terraform HCL, GitOps MRs)
|
||||
|
||||
### 3. **Drift Prevention**
|
||||
- **Tools section**: Exact MCP tool names prevent registry/source mismatch
|
||||
- **Entrypoint specification**: Prevents confusion between modules and runnable servers
|
||||
- **File references**: Uses actual file names (e.g., `invariant_checker_py.py`)
|
||||
|
||||
### 4. **Machine-Checkable Contract**
|
||||
The registry now serves as a **verifiable contract** between:
|
||||
- **Documentation**: What capabilities are claimed
|
||||
- **Implementation**: What tools are actually exposed
|
||||
- **Operations**: What side effects and auth are required
|
||||
|
||||
## 🎯 Registry Structure
|
||||
|
||||
### MCP Server Template
|
||||
```yaml
|
||||
server_name:
|
||||
module: "exact.python.module.path"
|
||||
entrypoint: "runnable.server.path"
|
||||
purpose: "clear operational purpose"
|
||||
tools: ["exact_tool_name (operation_type)"]
|
||||
auth_env: ["REQUIRED_VARS"]
|
||||
side_effects: "read-only | propose | mutate"
|
||||
outputs: ["json", "terraform_hcl", "gitops_mr"]
|
||||
```
|
||||
|
||||
### GitOps Tool Template
|
||||
```yaml
|
||||
tool_name:
|
||||
file: "exact/file/path.py"
|
||||
purpose: "specific operational function"
|
||||
side_effects: "creates MRs | applies changes | generates reports"
|
||||
outputs: ["terraform_apply", "gitops_mr", "anomaly_report"]
|
||||
```
|
||||
|
||||
## 🔒 Audit-Grade Features
|
||||
|
||||
### Compliance Validation
|
||||
- **Tool enumeration**: Every MCP tool is explicitly listed
|
||||
- **Access control**: Auth requirements clearly documented
|
||||
- **Change tracking**: Versioned registry with generation timestamps
|
||||
|
||||
### Operational Transparency
|
||||
- **Side effects**: Clear about mutating vs. read-only operations
|
||||
- **Output formats**: Specific about what each component produces
|
||||
- **Dependencies**: Environmental requirements explicitly stated
|
||||
|
||||
### Drift Detection
|
||||
- **File references**: Uses actual file names to prevent rename drift
|
||||
- **Module paths**: Exact Python module paths prevent import confusion
|
||||
- **Tool names**: Exact MCP tool names prevent capability mismatch
|
||||
|
||||
## 🚀 Usage Examples
|
||||
|
||||
### For Auditors
|
||||
```bash
|
||||
# Verify MCP server capabilities match documentation
|
||||
cat capability_registry_v2.json | jq '.mcp_servers.cloudflare_safe.tools'
|
||||
|
||||
# Check operational requirements
|
||||
cat capability_registry_v2.json | jq '.mcp_servers.cloudflare_safe.auth_env'
|
||||
```
|
||||
|
||||
### For Developers
|
||||
```bash
|
||||
# Validate new MCP server against registry template
|
||||
python3 generate_capability_registry_v2.py
|
||||
|
||||
# Check for capability drift
|
||||
diff capability_registry_v2.json capability_registry.json
|
||||
```
|
||||
|
||||
### For Operations
|
||||
```bash
|
||||
# Verify side effects before deployment
|
||||
cat capability_registry_v2.json | jq '.mcp_servers.cloudflare_safe.side_effects'
|
||||
|
||||
# Check output formats for integration
|
||||
cat capability_registry_v2.json | jq '.gitops_tools.waf_rule_proposer.outputs'
|
||||
```
|
||||
|
||||
## 📊 Registry Files Generated
|
||||
|
||||
1. **`capability_registry_v2.json`** - Machine-readable contract
|
||||
2. **`CAPABILITY_REGISTRY_V2.md`** - Human-readable documentation
|
||||
3. **`generate_capability_registry_v2.py`** - Regeneration script
|
||||
|
||||
## 🎯 Next Steps
|
||||
|
||||
### Continuous Validation
|
||||
- Add CI check to validate MCP tool names against registry
|
||||
- Automated drift detection between registry and source code
|
||||
- Periodic registry regeneration as capabilities evolve
|
||||
|
||||
### Extended Metadata
|
||||
- Add performance characteristics (timeouts, rate limits)
|
||||
- Include error handling patterns
|
||||
- Add recovery procedures for failed operations
|
||||
|
||||
### Integration Testing
|
||||
- Use registry to generate comprehensive test suites
|
||||
- Validate auth/env requirements in test environment
|
||||
- Verify side effects and outputs match expectations
|
||||
|
||||
This enhanced registry transforms the Cloudflare control plane from **documented infrastructure** to **verifiable, auditable, and drift-resistant infrastructure**.
|
||||
@@ -109,6 +109,8 @@ A local MCP server is registered in `opencode.jsonc` as `waf_intel`:
|
||||
}
|
||||
```
|
||||
|
||||
`waf_intel_mcp.py` delegates to the in-repo MCP stdio JSON-RPC implementation (`mcp.waf_intelligence.mcp_server`), so it does not require installing a separate Python MCP SDK.
|
||||
|
||||
The `security-audit` agent has `waf_intel` enabled in its tools section:
|
||||
|
||||
```jsonc
|
||||
|
||||
326
USAGE_GUIDE.md
Normal file
326
USAGE_GUIDE.md
Normal file
@@ -0,0 +1,326 @@
|
||||
# Cloudflare MCP Tools Usage Guide
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### 1. Configure Environment
|
||||
|
||||
```bash
|
||||
# Copy and edit the environment file
|
||||
cp .env.example .env
|
||||
|
||||
# Edit with your Cloudflare credentials
|
||||
nano .env
|
||||
```
|
||||
|
||||
**Required Credentials:**
|
||||
- `CLOUDFLARE_API_TOKEN`: API token with Zone:Read, Zone:Write permissions
|
||||
- `CLOUDFLARE_ACCOUNT_ID`: Your Cloudflare account ID
|
||||
|
||||
### 2. Load Environment
|
||||
|
||||
```bash
|
||||
# Source the environment
|
||||
source .env
|
||||
|
||||
# Set Python path for MCP servers
|
||||
export PYTHONPATH="/Users/sovereign/work-core"
|
||||
```
|
||||
|
||||
## 🔧 Available MCP Tools
|
||||
|
||||
### Cloudflare Safe MCP (`cloudflare.mcp.cloudflare_safe`)
|
||||
|
||||
**Tools for managing Cloudflare infrastructure:**
|
||||
|
||||
#### 1. Take Snapshot of Current State
|
||||
```bash
|
||||
python3 -c "
|
||||
from cloudflare.mcp.cloudflare_safe.server import CloudflareServer
|
||||
import os
|
||||
|
||||
# Set environment
|
||||
os.environ['CLOUDFLARE_API_TOKEN'] = 'your_token'
|
||||
os.environ['CLOUDFLARE_ACCOUNT_ID'] = 'your_account_id'
|
||||
|
||||
server = CloudflareServer()
|
||||
result = server.cf_snapshot(scopes=['zones', 'tunnels', 'access_apps'])
|
||||
print('Snapshot ID:', result['data']['snapshot_id'])
|
||||
print('Summary:', result['summary'])
|
||||
"
|
||||
```
|
||||
|
||||
#### 2. List DNS Zones
|
||||
```bash
|
||||
python3 -c "
|
||||
from cloudflare.mcp.cloudflare_safe.server import CloudflareServer
|
||||
import os
|
||||
|
||||
os.environ['CLOUDFLARE_API_TOKEN'] = 'your_token'
|
||||
os.environ['CLOUDFLARE_ACCOUNT_ID'] = 'your_account_id'
|
||||
|
||||
server = CloudflareServer()
|
||||
result = server.cf_snapshot(scopes=['zones'])
|
||||
zones = result['data']['counts']['zones']
|
||||
print(f'Found {zones} DNS zones')
|
||||
"
|
||||
```
|
||||
|
||||
#### 3. Check Tunnel Status
|
||||
```bash
|
||||
python3 -c "
|
||||
from cloudflare.mcp.cloudflare_safe.server import CloudflareServer
|
||||
import os
|
||||
|
||||
os.environ['CLOUDFLARE_API_TOKEN'] = 'your_token'
|
||||
os.environ['CLOUDFLARE_ACCOUNT_ID'] = 'your_account_id'
|
||||
|
||||
server = CloudflareServer()
|
||||
result = server.cf_tunnel_status()
|
||||
print('Tunnel status:', result)
|
||||
"
|
||||
```
|
||||
|
||||
### WAF Intelligence MCP (`cloudflare.mcp.waf_intelligence.mcp_server`)
|
||||
|
||||
**Tools for security analysis and rule generation:**
|
||||
|
||||
#### 1. Analyze WAF Configuration
|
||||
```bash
|
||||
python3 -m cloudflare.mcp.waf_intelligence.mcp_server --file terraform/waf.tf --format text
|
||||
```
|
||||
|
||||
#### 2. Generate Security Rules
|
||||
```bash
|
||||
python3 -c "
|
||||
from cloudflare.mcp.waf_intelligence.orchestrator import WAFIntelligence
|
||||
|
||||
waf_intel = WAFIntelligence()
|
||||
analysis = waf_intel.analyze_and_recommend('terraform/waf.tf')
|
||||
print('Security recommendations:', analysis)
|
||||
"
|
||||
```
|
||||
|
||||
## 🌐 Setting Up Domains
|
||||
|
||||
### 1. Configure DNS Records via Terraform
|
||||
|
||||
**Example DNS Configuration:**
|
||||
|
||||
```hcl
|
||||
# terraform/dns.tf
|
||||
resource "cloudflare_zone" "domains" {
|
||||
for_each = toset(["vaultmesh.org", "offsec.global"])
|
||||
|
||||
zone = each.key
|
||||
plan = "free"
|
||||
}
|
||||
|
||||
resource "cloudflare_record" "root_a" {
|
||||
for_each = cloudflare_zone.domains
|
||||
|
||||
zone_id = each.value.id
|
||||
name = "@"
|
||||
value = "192.168.1.100" # Your server IP
|
||||
type = "A"
|
||||
proxied = true
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Apply DNS Configuration
|
||||
|
||||
```bash
|
||||
# Initialize Terraform
|
||||
terraform init
|
||||
|
||||
# Plan changes
|
||||
terraform plan
|
||||
|
||||
# Apply DNS configuration
|
||||
terraform apply
|
||||
```
|
||||
|
||||
## 🛡️ Configuring WAF Security
|
||||
|
||||
### 1. Basic WAF Rules
|
||||
|
||||
```hcl
|
||||
# terraform/waf.tf
|
||||
resource "cloudflare_ruleset" "security_rules" {
|
||||
for_each = cloudflare_zone.domains
|
||||
|
||||
zone_id = each.value.id
|
||||
name = "Security Rules"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_custom"
|
||||
|
||||
# Block admin access from untrusted IPs
|
||||
rules {
|
||||
action = "block"
|
||||
expression = "(http.request.uri.path contains '/admin') and not (ip.src in {192.168.1.1 10.0.0.1})"
|
||||
description = "Block admin access from untrusted IPs"
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Enable Managed WAF
|
||||
|
||||
```hcl
|
||||
resource "cloudflare_ruleset" "managed_waf" {
|
||||
for_each = cloudflare_zone.domains
|
||||
|
||||
zone_id = each.value.id
|
||||
name = "Managed WAF"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_managed"
|
||||
|
||||
# Cloudflare Managed Ruleset
|
||||
rules {
|
||||
action = "execute"
|
||||
action_parameters {
|
||||
id = "efb7b8c949ac4650a09736fc376e9aee"
|
||||
}
|
||||
expression = "true"
|
||||
description = "Execute Cloudflare Managed Ruleset"
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 🌉 Setting Up Cloudflare Tunnels
|
||||
|
||||
### 1. Configure Tunnels
|
||||
|
||||
```hcl
|
||||
# terraform/tunnels.tf
|
||||
resource "cloudflare_tunnel" "vaultmesh" {
|
||||
account_id = local.account_id
|
||||
name = "vaultmesh-tunnel"
|
||||
secret = var.tunnel_secret_vaultmesh
|
||||
}
|
||||
|
||||
resource "cloudflare_tunnel_config" "vaultmesh" {
|
||||
account_id = local.account_id
|
||||
tunnel_id = cloudflare_tunnel.vaultmesh.id
|
||||
|
||||
config {
|
||||
# API endpoint
|
||||
ingress_rule {
|
||||
hostname = "api.vaultmesh.org"
|
||||
service = "http://localhost:8080"
|
||||
}
|
||||
|
||||
# Dashboard
|
||||
ingress_rule {
|
||||
hostname = "dash.vaultmesh.org"
|
||||
service = "http://localhost:3000"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Generate Tunnel Secrets
|
||||
|
||||
```bash
|
||||
# Generate secure tunnel secrets
|
||||
openssl rand -base64 32
|
||||
|
||||
# Add to your .env file
|
||||
TUNNEL_SECRET_VAULTMESH="generated_secret_here"
|
||||
```
|
||||
|
||||
## 🔍 Monitoring and Validation
|
||||
|
||||
### 1. Check Current State
|
||||
|
||||
```bash
|
||||
# Use the invariant checker to validate configuration
|
||||
python3 scripts/invariant_checker_py.py
|
||||
```
|
||||
|
||||
### 2. Monitor Tunnel Health
|
||||
|
||||
```bash
|
||||
# Check tunnel status via MCP
|
||||
python3 -c "
|
||||
from cloudflare.mcp.cloudflare_safe.server import CloudflareServer
|
||||
import os
|
||||
|
||||
os.environ.update({
|
||||
'CLOUDFLARE_API_TOKEN': 'your_token',
|
||||
'CLOUDFLARE_ACCOUNT_ID': 'your_account_id'
|
||||
})
|
||||
|
||||
server = CloudflareServer()
|
||||
status = server.cf_tunnel_status()
|
||||
print('Tunnel health:', status)
|
||||
"
|
||||
```
|
||||
|
||||
## 🚨 Common Operations
|
||||
|
||||
### Adding New Domain
|
||||
|
||||
1. **Add to Terraform zones list**
|
||||
2. **Run `terraform apply`**
|
||||
3. **Verify DNS propagation**
|
||||
4. **Configure WAF rules**
|
||||
|
||||
### Updating Security Rules
|
||||
|
||||
1. **Modify `terraform/waf.tf`**
|
||||
2. **Run `terraform plan` to preview**
|
||||
3. **Apply with `terraform apply`**
|
||||
4. **Validate with WAF Intelligence MCP**
|
||||
|
||||
### Tunnel Management
|
||||
|
||||
1. **Generate new tunnel secret**
|
||||
2. **Update Terraform configuration**
|
||||
3. **Apply changes**
|
||||
4. **Verify connectivity**
|
||||
|
||||
## 📊 Best Practices
|
||||
|
||||
### Security
|
||||
- Use least-privilege API tokens
|
||||
- Enable 2FA on Cloudflare account
|
||||
- Regular security audits with WAF Intelligence
|
||||
- Monitor access logs
|
||||
|
||||
### Operations
|
||||
- Test changes in staging first
|
||||
- Use Terraform for all infrastructure changes
|
||||
- Regular backups of Terraform state
|
||||
- Monitor tunnel health
|
||||
|
||||
### Monitoring
|
||||
- Set up Cloudflare analytics
|
||||
- Monitor WAF rule effectiveness
|
||||
- Track DNS resolution times
|
||||
- Alert on security events
|
||||
|
||||
## 🆘 Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**API Token Errors**
|
||||
```bash
|
||||
# Verify token permissions
|
||||
curl -X GET "https://api.cloudflare.com/client/v4/user/tokens/verify" \
|
||||
-H "Authorization: Bearer $CLOUDFLARE_API_TOKEN"
|
||||
```
|
||||
|
||||
**Tunnel Connectivity**
|
||||
```bash
|
||||
# Check cloudflared service status
|
||||
cloudflared tunnel list
|
||||
```
|
||||
|
||||
**DNS Issues**
|
||||
```bash
|
||||
# Verify DNS resolution
|
||||
dig yourdomain.com
|
||||
```
|
||||
|
||||
This guide provides the foundation for managing your Cloudflare infrastructure using the MCP tools. Start with basic DNS setup, then progressively add WAF rules and tunnels as needed.
|
||||
185
capability_registry.json
Normal file
185
capability_registry.json
Normal file
@@ -0,0 +1,185 @@
|
||||
{
|
||||
"metadata": {
|
||||
"generated_at": "2025-12-18T02:19:38.165161+00:00",
|
||||
"version": "1.0.0",
|
||||
"scope": "Cloudflare Control Plane"
|
||||
},
|
||||
"mcp_servers": {
|
||||
"cloudflare_safe": {
|
||||
"module": "cloudflare.mcp.cloudflare_safe",
|
||||
"purpose": "Secure Cloudflare API operations",
|
||||
"capabilities": [
|
||||
"dns_record_management",
|
||||
"waf_rule_configuration",
|
||||
"tunnel_health_monitoring",
|
||||
"zone_analytics_query",
|
||||
"terraform_state_synchronization"
|
||||
],
|
||||
"security": {
|
||||
"token_redaction": true,
|
||||
"error_handling": true,
|
||||
"rate_limiting": true
|
||||
}
|
||||
},
|
||||
"waf_intelligence": {
|
||||
"module": "cloudflare.mcp.waf_intelligence",
|
||||
"purpose": "WAF rule analysis and synthesis",
|
||||
"capabilities": [
|
||||
"waf_config_analysis",
|
||||
"threat_intelligence_integration",
|
||||
"compliance_mapping",
|
||||
"rule_gap_identification",
|
||||
"terraform_ready_rule_generation"
|
||||
],
|
||||
"intelligence": {
|
||||
"ml_classification": true,
|
||||
"threat_intel": true,
|
||||
"compliance_frameworks": [
|
||||
"PCI-DSS 6.6",
|
||||
"OWASP-ASVS 13"
|
||||
]
|
||||
}
|
||||
},
|
||||
"oracle_answer": {
|
||||
"module": "cloudflare.mcp.oracle_answer",
|
||||
"purpose": "Security decision support",
|
||||
"capabilities": [
|
||||
"security_classification",
|
||||
"routing_decision_support",
|
||||
"threat_assessment",
|
||||
"pre_execution_screening"
|
||||
],
|
||||
"integration": {
|
||||
"layer0_framework": true,
|
||||
"shadow_classifier": true,
|
||||
"preboot_logging": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"terraform_resources": {
|
||||
"dns_management": {
|
||||
"files": [
|
||||
"dns.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_record",
|
||||
"cloudflare_zone"
|
||||
],
|
||||
"capabilities": [
|
||||
"automated_dns_provisioning",
|
||||
"spf_dmarc_mx_configuration",
|
||||
"tunnel_based_routing",
|
||||
"proxied_record_management"
|
||||
]
|
||||
},
|
||||
"waf_security": {
|
||||
"files": [
|
||||
"waf.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_ruleset",
|
||||
"cloudflare_bot_management"
|
||||
],
|
||||
"capabilities": [
|
||||
"custom_waf_rules",
|
||||
"managed_ruleset_integration",
|
||||
"bot_management",
|
||||
"rate_limiting",
|
||||
"country_blocking"
|
||||
]
|
||||
},
|
||||
"tunnel_infrastructure": {
|
||||
"files": [
|
||||
"tunnels.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_tunnel",
|
||||
"cloudflare_tunnel_config"
|
||||
],
|
||||
"capabilities": [
|
||||
"multi_service_tunnel_routing",
|
||||
"ingress_rule_management",
|
||||
"health_monitoring",
|
||||
"credential_rotation"
|
||||
]
|
||||
}
|
||||
},
|
||||
"gitops_tools": {
|
||||
"waf_rule_proposer": {
|
||||
"file": "gitops/waf_rule_proposer.py",
|
||||
"purpose": "Automated WAF rule generation",
|
||||
"capabilities": [
|
||||
"threat_intel_driven_rules",
|
||||
"gitlab_ci_integration",
|
||||
"automated_mr_creation",
|
||||
"compliance_mapping"
|
||||
]
|
||||
},
|
||||
"invariant_checker": {
|
||||
"file": "scripts/invariant_checker_py.py",
|
||||
"purpose": "Real-time state validation",
|
||||
"capabilities": [
|
||||
"dns_integrity_checks",
|
||||
"waf_compliance_validation",
|
||||
"tunnel_health_monitoring",
|
||||
"drift_detection"
|
||||
]
|
||||
},
|
||||
"drift_guardian": {
|
||||
"file": "scripts/drift_guardian_py.py",
|
||||
"purpose": "Automated remediation",
|
||||
"capabilities": [
|
||||
"state_reconciliation",
|
||||
"auto_remediation",
|
||||
"ops_notification"
|
||||
]
|
||||
}
|
||||
},
|
||||
"security_framework": {
|
||||
"layer0": {
|
||||
"components": [
|
||||
"entrypoint.py",
|
||||
"shadow_classifier.py",
|
||||
"preboot_logger.py"
|
||||
],
|
||||
"capabilities": [
|
||||
"pre_execution_security_classification",
|
||||
"threat_assessment",
|
||||
"security_event_logging",
|
||||
"routing_decision_support"
|
||||
],
|
||||
"classification_levels": [
|
||||
"catastrophic",
|
||||
"forbidden",
|
||||
"ambiguous",
|
||||
"blessed"
|
||||
]
|
||||
}
|
||||
},
|
||||
"operational_tools": {
|
||||
"systemd_services": {
|
||||
"services": [
|
||||
"autonomous-remediator",
|
||||
"drift-guardian",
|
||||
"tunnel-rotation"
|
||||
],
|
||||
"capabilities": [
|
||||
"continuous_monitoring",
|
||||
"automated_remediation",
|
||||
"scheduled_operations"
|
||||
]
|
||||
},
|
||||
"test_suites": {
|
||||
"suites": [
|
||||
"layer0_validation",
|
||||
"mcp_integration",
|
||||
"cloudflare_safe_ingress"
|
||||
],
|
||||
"capabilities": [
|
||||
"security_classification_testing",
|
||||
"mcp_server_validation",
|
||||
"api_integration_testing"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
243
capability_registry_v2.json
Normal file
243
capability_registry_v2.json
Normal file
@@ -0,0 +1,243 @@
|
||||
{
|
||||
"metadata": {
|
||||
"generated_at": "2025-12-18T02:38:01.740122+00:00",
|
||||
"version": "1.0.1",
|
||||
"scope": "Cloudflare Control Plane"
|
||||
},
|
||||
"mcp_servers": {
|
||||
"cloudflare_safe": {
|
||||
"module": "cloudflare.mcp.cloudflare_safe",
|
||||
"entrypoint": "cloudflare.mcp.cloudflare_safe",
|
||||
"purpose": "Secure Cloudflare API operations",
|
||||
"tools": [
|
||||
"cf_snapshot (read/write token required)",
|
||||
"cf_refresh (write token required)",
|
||||
"cf_config_diff (read; requires snapshot_id)",
|
||||
"cf_export_config (read)",
|
||||
"cf_tunnel_status (read)",
|
||||
"cf_tunnel_ingress_summary (read)",
|
||||
"cf_access_policy_list (read)"
|
||||
],
|
||||
"auth_env": [
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"CLOUDFLARE_ACCOUNT_ID"
|
||||
],
|
||||
"side_effects": "read-only unless token present; cf_refresh/cf_snapshot are mutating",
|
||||
"outputs": [
|
||||
"json",
|
||||
"terraform_hcl"
|
||||
],
|
||||
"capabilities": [
|
||||
"dns_record_management",
|
||||
"waf_rule_configuration",
|
||||
"tunnel_health_monitoring",
|
||||
"zone_analytics_query",
|
||||
"terraform_state_synchronization"
|
||||
],
|
||||
"security": {
|
||||
"token_redaction": true,
|
||||
"error_handling": true,
|
||||
"rate_limiting": true
|
||||
}
|
||||
},
|
||||
"waf_intelligence": {
|
||||
"module": "cloudflare.mcp.waf_intelligence",
|
||||
"entrypoint": "cloudflare.mcp.waf_intelligence.mcp_server",
|
||||
"purpose": "WAF rule analysis and synthesis",
|
||||
"tools": [
|
||||
"waf_capabilities (read)",
|
||||
"waf_analyze (read)",
|
||||
"waf_assess (read)",
|
||||
"waf_generate_gitops_proposals (propose)"
|
||||
],
|
||||
"auth_env": [],
|
||||
"side_effects": "propose-only; generates GitOps proposals",
|
||||
"outputs": [
|
||||
"json",
|
||||
"terraform_hcl",
|
||||
"gitops_mr"
|
||||
],
|
||||
"capabilities": [
|
||||
"waf_config_analysis",
|
||||
"threat_intelligence_integration",
|
||||
"compliance_mapping",
|
||||
"rule_gap_identification",
|
||||
"terraform_ready_rule_generation"
|
||||
],
|
||||
"intelligence": {
|
||||
"ml_classification": true,
|
||||
"threat_intel": true,
|
||||
"compliance_frameworks": [
|
||||
"PCI-DSS 6.6",
|
||||
"OWASP-ASVS 13"
|
||||
]
|
||||
}
|
||||
},
|
||||
"oracle_answer": {
|
||||
"module": "cloudflare.mcp.oracle_answer",
|
||||
"entrypoint": "cloudflare.mcp.oracle_answer",
|
||||
"purpose": "Security decision support",
|
||||
"tools": [
|
||||
"oracle_answer (read)"
|
||||
],
|
||||
"auth_env": [],
|
||||
"side_effects": "read-only; security classification only",
|
||||
"outputs": [
|
||||
"json",
|
||||
"security_classification"
|
||||
],
|
||||
"capabilities": [
|
||||
"security_classification",
|
||||
"routing_decision_support",
|
||||
"threat_assessment",
|
||||
"pre_execution_screening"
|
||||
],
|
||||
"integration": {
|
||||
"layer0_framework": true,
|
||||
"shadow_classifier": true,
|
||||
"preboot_logging": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"terraform_resources": {
|
||||
"dns_management": {
|
||||
"files": [
|
||||
"dns.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_record",
|
||||
"cloudflare_zone"
|
||||
],
|
||||
"capabilities": [
|
||||
"automated_dns_provisioning",
|
||||
"spf_dmarc_mx_configuration",
|
||||
"tunnel_based_routing",
|
||||
"proxied_record_management"
|
||||
]
|
||||
},
|
||||
"waf_security": {
|
||||
"files": [
|
||||
"waf.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_ruleset",
|
||||
"cloudflare_bot_management"
|
||||
],
|
||||
"capabilities": [
|
||||
"custom_waf_rules",
|
||||
"managed_ruleset_integration",
|
||||
"bot_management",
|
||||
"rate_limiting",
|
||||
"country_blocking"
|
||||
]
|
||||
},
|
||||
"tunnel_infrastructure": {
|
||||
"files": [
|
||||
"tunnels.tf"
|
||||
],
|
||||
"resources": [
|
||||
"cloudflare_tunnel",
|
||||
"cloudflare_tunnel_config"
|
||||
],
|
||||
"capabilities": [
|
||||
"multi_service_tunnel_routing",
|
||||
"ingress_rule_management",
|
||||
"health_monitoring",
|
||||
"credential_rotation"
|
||||
]
|
||||
}
|
||||
},
|
||||
"gitops_tools": {
|
||||
"waf_rule_proposer": {
|
||||
"file": "gitops/waf_rule_proposer.py",
|
||||
"purpose": "Automated WAF rule generation",
|
||||
"side_effects": "creates GitLab merge requests",
|
||||
"outputs": [
|
||||
"terraform_hcl",
|
||||
"gitops_mr"
|
||||
],
|
||||
"capabilities": [
|
||||
"threat_intel_driven_rules",
|
||||
"gitlab_ci_integration",
|
||||
"automated_mr_creation",
|
||||
"compliance_mapping"
|
||||
]
|
||||
},
|
||||
"invariant_checker": {
|
||||
"file": "scripts/invariant_checker_py.py",
|
||||
"purpose": "Real-time state validation",
|
||||
"side_effects": "generates anomaly reports",
|
||||
"outputs": [
|
||||
"json",
|
||||
"anomaly_report"
|
||||
],
|
||||
"capabilities": [
|
||||
"dns_integrity_checks",
|
||||
"waf_compliance_validation",
|
||||
"tunnel_health_monitoring",
|
||||
"drift_detection"
|
||||
]
|
||||
},
|
||||
"drift_guardian": {
|
||||
"file": "scripts/drift_guardian_py.py",
|
||||
"purpose": "Automated remediation",
|
||||
"side_effects": "applies Terraform changes",
|
||||
"outputs": [
|
||||
"terraform_apply",
|
||||
"remediation_report"
|
||||
],
|
||||
"capabilities": [
|
||||
"state_reconciliation",
|
||||
"auto_remediation",
|
||||
"ops_notification"
|
||||
]
|
||||
}
|
||||
},
|
||||
"security_framework": {
|
||||
"layer0": {
|
||||
"components": [
|
||||
"entrypoint.py",
|
||||
"shadow_classifier.py",
|
||||
"preboot_logger.py"
|
||||
],
|
||||
"capabilities": [
|
||||
"pre_execution_security_classification",
|
||||
"threat_assessment",
|
||||
"security_event_logging",
|
||||
"routing_decision_support"
|
||||
],
|
||||
"classification_levels": [
|
||||
"catastrophic",
|
||||
"forbidden",
|
||||
"ambiguous",
|
||||
"blessed"
|
||||
]
|
||||
}
|
||||
},
|
||||
"operational_tools": {
|
||||
"systemd_services": {
|
||||
"services": [
|
||||
"autonomous-remediator",
|
||||
"drift-guardian",
|
||||
"tunnel-rotation"
|
||||
],
|
||||
"capabilities": [
|
||||
"continuous_monitoring",
|
||||
"automated_remediation",
|
||||
"scheduled_operations"
|
||||
]
|
||||
},
|
||||
"test_suites": {
|
||||
"suites": [
|
||||
"layer0_validation",
|
||||
"mcp_integration",
|
||||
"cloudflare_safe_ingress"
|
||||
],
|
||||
"capabilities": [
|
||||
"security_classification_testing",
|
||||
"mcp_server_validation",
|
||||
"api_integration_testing"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
95
ci_check_entrypoints.py
Executable file
95
ci_check_entrypoints.py
Executable file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI Entrypoint Sanity Check
|
||||
|
||||
Validates that all MCP server entrypoints are runnable.
|
||||
Fails CI if any entrypoint has import or startup errors.
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_registry_entrypoints():
|
||||
"""Load entrypoints from capability registry."""
|
||||
with open("capability_registry_v2.json", "r") as f:
|
||||
registry = json.load(f)
|
||||
|
||||
entrypoints = {}
|
||||
for server_name, server_info in registry["mcp_servers"].items():
|
||||
entrypoints[server_name] = server_info["entrypoint"]
|
||||
|
||||
return entrypoints
|
||||
|
||||
|
||||
def check_entrypoint(server_name: str, entrypoint: str) -> tuple[bool, str]:
|
||||
"""Check if an entrypoint is runnable."""
|
||||
try:
|
||||
# Test with --help flag or equivalent
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = "/Users/sovereign/work-core"
|
||||
|
||||
result = subprocess.run(
|
||||
["python3", "-m", entrypoint, "--help"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
env=env,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return True, f"✅ {server_name}: Entrypoint '{entrypoint}' is runnable"
|
||||
else:
|
||||
return (
|
||||
False,
|
||||
f"❌ {server_name}: Entrypoint '{entrypoint}' failed with exit code {result.returncode}\n{result.stderr}",
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, f"❌ {server_name}: Entrypoint '{entrypoint}' timed out"
|
||||
except FileNotFoundError:
|
||||
return False, f"❌ {server_name}: Entrypoint '{entrypoint}' not found"
|
||||
except Exception as e:
|
||||
return (
|
||||
False,
|
||||
f"❌ {server_name}: Entrypoint '{entrypoint}' failed with error: {e}",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main CI check function."""
|
||||
print("🔍 CI Entrypoint Sanity Check")
|
||||
print("=" * 50)
|
||||
|
||||
entrypoints = get_registry_entrypoints()
|
||||
errors = []
|
||||
successes = []
|
||||
|
||||
for server_name, entrypoint in entrypoints.items():
|
||||
success, message = check_entrypoint(server_name, entrypoint)
|
||||
if success:
|
||||
successes.append(message)
|
||||
else:
|
||||
errors.append(message)
|
||||
|
||||
# Print results
|
||||
for success in successes:
|
||||
print(success)
|
||||
|
||||
for error in errors:
|
||||
print(error)
|
||||
|
||||
if errors:
|
||||
print(f"\n❌ {len(errors)} entrypoint(s) failed")
|
||||
print("💡 Fix: Update capability_registry_v2.json with correct entrypoints")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"\n✅ All {len(successes)} entrypoints are runnable")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
140
ci_check_tool_names.py
Executable file
140
ci_check_tool_names.py
Executable file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI Tool Name Parity Check
|
||||
|
||||
Validates that MCP server tool names match the capability registry.
|
||||
Fails CI if:
|
||||
- A tool exists but isn't registered
|
||||
- A registered tool no longer exists
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_registry_tools():
|
||||
"""Load tool names from capability registry."""
|
||||
with open("capability_registry_v2.json", "r") as f:
|
||||
registry = json.load(f)
|
||||
|
||||
registry_tools = {}
|
||||
for server_name, server_info in registry["mcp_servers"].items():
|
||||
# Extract base tool names (remove operation type)
|
||||
tools = []
|
||||
for tool_desc in server_info["tools"]:
|
||||
tool_name = tool_desc.split(" (")[0] # Remove "(operation_type)"
|
||||
tools.append(tool_name)
|
||||
registry_tools[server_name] = set(tools)
|
||||
|
||||
return registry_tools
|
||||
|
||||
|
||||
def get_actual_tools():
|
||||
"""Get actual tool names from MCP servers by examining source code."""
|
||||
actual_tools = {}
|
||||
|
||||
# Extract tools from Cloudflare Safe server source
|
||||
try:
|
||||
with open("mcp/cloudflare_safe/server.py", "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# Look for tool function definitions
|
||||
import re
|
||||
|
||||
tool_pattern = r"def (cf_\w+)\("
|
||||
tools_found = set(re.findall(tool_pattern, content))
|
||||
|
||||
# Filter out internal functions
|
||||
valid_tools = {tool for tool in tools_found if not tool.startswith("_")}
|
||||
actual_tools["cloudflare_safe"] = valid_tools
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not extract tools from cloudflare_safe: {e}")
|
||||
|
||||
# Extract tools from WAF Intelligence server source
|
||||
try:
|
||||
with open("mcp/waf_intelligence/mcp_server.py", "r") as f:
|
||||
content = f.read()
|
||||
|
||||
tool_pattern = r"def (waf_\w+)\("
|
||||
tools_found = set(re.findall(tool_pattern, content))
|
||||
|
||||
valid_tools = {tool for tool in tools_found if not tool.startswith("_")}
|
||||
actual_tools["waf_intelligence"] = valid_tools
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not extract tools from waf_intelligence: {e}")
|
||||
|
||||
# Extract tools from Oracle Answer server source
|
||||
try:
|
||||
with open("mcp/oracle_answer/server.py", "r") as f:
|
||||
content = f.read()
|
||||
|
||||
tool_pattern = r"def (\w+)\("
|
||||
tools_found = set(re.findall(tool_pattern, content))
|
||||
|
||||
# Look for oracle_answer specifically
|
||||
oracle_tools = {tool for tool in tools_found if "oracle" in tool.lower()}
|
||||
actual_tools["oracle_answer"] = oracle_tools
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not extract tools from oracle_answer: {e}")
|
||||
|
||||
return actual_tools
|
||||
|
||||
|
||||
def check_tool_parity():
|
||||
"""Compare registry tools with actual tools."""
|
||||
registry_tools = get_registry_tools()
|
||||
actual_tools = get_actual_tools()
|
||||
|
||||
errors = []
|
||||
|
||||
for server_name in set(registry_tools.keys()) | set(actual_tools.keys()):
|
||||
reg_tools = registry_tools.get(server_name, set())
|
||||
act_tools = actual_tools.get(server_name, set())
|
||||
|
||||
# Check for tools in registry but not in actual
|
||||
missing_in_actual = reg_tools - act_tools
|
||||
if missing_in_actual:
|
||||
errors.append(
|
||||
f"❌ {server_name}: Tools registered but not found: {missing_in_actual}"
|
||||
)
|
||||
|
||||
# Check for tools in actual but not in registry
|
||||
missing_in_registry = act_tools - reg_tools
|
||||
if missing_in_registry:
|
||||
errors.append(
|
||||
f"❌ {server_name}: Tools found but not registered: {missing_in_registry}"
|
||||
)
|
||||
|
||||
# Report parity
|
||||
if not missing_in_actual and not missing_in_registry:
|
||||
print(f"✅ {server_name}: Tool parity verified ({len(reg_tools)} tools)")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main():
|
||||
"""Main CI check function."""
|
||||
print("🔍 CI Tool Name Parity Check")
|
||||
print("=" * 50)
|
||||
|
||||
errors = check_tool_parity()
|
||||
|
||||
if errors:
|
||||
print("\n❌ Registry drift detected:")
|
||||
for error in errors:
|
||||
print(error)
|
||||
print(
|
||||
"\n💡 Fix: Update capability_registry_v2.json to match actual MCP server tools"
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("\n✅ All MCP server tools match capability registry")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
350
generate_capability_registry.py
Normal file
350
generate_capability_registry.py
Normal file
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cloudflare Control Plane Capability Registry Generator
|
||||
|
||||
Generates a machine-readable registry of all MCP server capabilities,
|
||||
Terraform resources, and operational tools for auditability and documentation.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Any
|
||||
|
||||
# Registry structure
|
||||
CAPABILITY_REGISTRY = {
|
||||
"metadata": {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"version": "1.0.0",
|
||||
"scope": "Cloudflare Control Plane",
|
||||
},
|
||||
"mcp_servers": {},
|
||||
"terraform_resources": {},
|
||||
"gitops_tools": {},
|
||||
"security_framework": {},
|
||||
"operational_tools": {},
|
||||
}
|
||||
|
||||
# MCP Server capabilities (from analysis)
|
||||
MCP_CAPABILITIES = {
|
||||
"cloudflare_safe": {
|
||||
"module": "cloudflare.mcp.cloudflare_safe",
|
||||
"entrypoint": "cloudflare.mcp.cloudflare_safe",
|
||||
"purpose": "Secure Cloudflare API operations",
|
||||
"tools": [
|
||||
"cf_snapshot (read/write token required)",
|
||||
"cf_refresh (write token required)",
|
||||
"cf_config_diff (read; requires snapshot_id)",
|
||||
"cf_export_config (read)",
|
||||
"cf_tunnel_status (read)",
|
||||
"cf_tunnel_ingress_summary (read)",
|
||||
"cf_access_policy_list (read)"
|
||||
],
|
||||
"auth_env": ["CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID"],
|
||||
"side_effects": "read-only unless token present; cf_refresh/cf_snapshot are mutating",
|
||||
"outputs": ["json", "terraform_hcl"],
|
||||
"capabilities": [
|
||||
"dns_record_management",
|
||||
"waf_rule_configuration",
|
||||
"tunnel_health_monitoring",
|
||||
"zone_analytics_query",
|
||||
"terraform_state_synchronization"
|
||||
],
|
||||
"security": {
|
||||
"token_redaction": True,
|
||||
"error_handling": True,
|
||||
"rate_limiting": True
|
||||
}
|
||||
},
|
||||
"waf_intelligence": {
|
||||
"module": "cloudflare.mcp.waf_intelligence",
|
||||
"entrypoint": "cloudflare.mcp.waf_intelligence.mcp_server",
|
||||
"purpose": "WAF rule analysis and synthesis",
|
||||
"tools": [
|
||||
"waf_capabilities (read)",
|
||||
"waf_analyze (read)",
|
||||
"waf_assess (read)",
|
||||
"waf_generate_gitops_proposals (propose)"
|
||||
],
|
||||
"auth_env": [],
|
||||
"side_effects": "propose-only; generates GitOps proposals",
|
||||
"outputs": ["json", "terraform_hcl", "gitops_mr"],
|
||||
"capabilities": [
|
||||
"waf_config_analysis",
|
||||
"threat_intelligence_integration",
|
||||
"compliance_mapping",
|
||||
"rule_gap_identification",
|
||||
"terraform_ready_rule_generation"
|
||||
],
|
||||
"intelligence": {
|
||||
"ml_classification": True,
|
||||
"threat_intel": True,
|
||||
"compliance_frameworks": ["PCI-DSS 6.6", "OWASP-ASVS 13"]
|
||||
}
|
||||
},
|
||||
"oracle_answer": {
|
||||
"module": "cloudflare.mcp.oracle_answer",
|
||||
"entrypoint": "cloudflare.mcp.oracle_answer",
|
||||
"purpose": "Security decision support",
|
||||
"tools": ["oracle_answer (read)"],
|
||||
"auth_env": [],
|
||||
"side_effects": "read-only; security classification only",
|
||||
"outputs": ["json", "security_classification"],
|
||||
"capabilities": [
|
||||
"security_classification",
|
||||
"routing_decision_support",
|
||||
"threat_assessment",
|
||||
"pre_execution_screening"
|
||||
],
|
||||
"integration": {
|
||||
"layer0_framework": True,
|
||||
"shadow_classifier": True,
|
||||
"preboot_logging": True
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"waf_intelligence": {
|
||||
"module": "cloudflare.mcp.waf_intelligence",
|
||||
"purpose": "WAF rule analysis and synthesis",
|
||||
"capabilities": [
|
||||
"waf_config_analysis",
|
||||
"threat_intelligence_integration",
|
||||
"compliance_mapping",
|
||||
"rule_gap_identification",
|
||||
"terraform_ready_rule_generation",
|
||||
],
|
||||
"intelligence": {
|
||||
"ml_classification": True,
|
||||
"threat_intel": True,
|
||||
"compliance_frameworks": ["PCI-DSS 6.6", "OWASP-ASVS 13"],
|
||||
},
|
||||
},
|
||||
"oracle_answer": {
|
||||
"module": "cloudflare.mcp.oracle_answer",
|
||||
"purpose": "Security decision support",
|
||||
"capabilities": [
|
||||
"security_classification",
|
||||
"routing_decision_support",
|
||||
"threat_assessment",
|
||||
"pre_execution_screening",
|
||||
],
|
||||
"integration": {
|
||||
"layer0_framework": True,
|
||||
"shadow_classifier": True,
|
||||
"preboot_logging": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Terraform resources (from analysis)
|
||||
TERRAFORM_RESOURCES = {
|
||||
"dns_management": {
|
||||
"files": ["dns.tf"],
|
||||
"resources": ["cloudflare_record", "cloudflare_zone"],
|
||||
"capabilities": [
|
||||
"automated_dns_provisioning",
|
||||
"spf_dmarc_mx_configuration",
|
||||
"tunnel_based_routing",
|
||||
"proxied_record_management",
|
||||
],
|
||||
},
|
||||
"waf_security": {
|
||||
"files": ["waf.tf"],
|
||||
"resources": ["cloudflare_ruleset", "cloudflare_bot_management"],
|
||||
"capabilities": [
|
||||
"custom_waf_rules",
|
||||
"managed_ruleset_integration",
|
||||
"bot_management",
|
||||
"rate_limiting",
|
||||
"country_blocking",
|
||||
],
|
||||
},
|
||||
"tunnel_infrastructure": {
|
||||
"files": ["tunnels.tf"],
|
||||
"resources": ["cloudflare_tunnel", "cloudflare_tunnel_config"],
|
||||
"capabilities": [
|
||||
"multi_service_tunnel_routing",
|
||||
"ingress_rule_management",
|
||||
"health_monitoring",
|
||||
"credential_rotation",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# GitOps tools
|
||||
GITOPS_TOOLS = {
|
||||
"waf_rule_proposer": {
|
||||
"file": "gitops/waf_rule_proposer.py",
|
||||
"purpose": "Automated WAF rule generation",
|
||||
"capabilities": [
|
||||
"threat_intel_driven_rules",
|
||||
"gitlab_ci_integration",
|
||||
"automated_mr_creation",
|
||||
"compliance_mapping",
|
||||
],
|
||||
},
|
||||
"invariant_checker": {
|
||||
"file": "scripts/invariant_checker_py.py",
|
||||
"purpose": "Real-time state validation",
|
||||
"capabilities": [
|
||||
"dns_integrity_checks",
|
||||
"waf_compliance_validation",
|
||||
"tunnel_health_monitoring",
|
||||
"drift_detection",
|
||||
],
|
||||
},
|
||||
"drift_guardian": {
|
||||
"file": "scripts/drift_guardian_py.py",
|
||||
"purpose": "Automated remediation",
|
||||
"capabilities": [
|
||||
"state_reconciliation",
|
||||
"auto_remediation",
|
||||
"ops_notification",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Security framework
|
||||
SECURITY_FRAMEWORK = {
|
||||
"layer0": {
|
||||
"components": ["entrypoint.py", "shadow_classifier.py", "preboot_logger.py"],
|
||||
"capabilities": [
|
||||
"pre_execution_security_classification",
|
||||
"threat_assessment",
|
||||
"security_event_logging",
|
||||
"routing_decision_support",
|
||||
],
|
||||
"classification_levels": ["catastrophic", "forbidden", "ambiguous", "blessed"],
|
||||
}
|
||||
}
|
||||
|
||||
# Operational tools
|
||||
OPERATIONAL_TOOLS = {
|
||||
"systemd_services": {
|
||||
"services": ["autonomous-remediator", "drift-guardian", "tunnel-rotation"],
|
||||
"capabilities": [
|
||||
"continuous_monitoring",
|
||||
"automated_remediation",
|
||||
"scheduled_operations",
|
||||
],
|
||||
},
|
||||
"test_suites": {
|
||||
"suites": ["layer0_validation", "mcp_integration", "cloudflare_safe_ingress"],
|
||||
"capabilities": [
|
||||
"security_classification_testing",
|
||||
"mcp_server_validation",
|
||||
"api_integration_testing",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def generate_registry():
|
||||
"""Generate the complete capability registry."""
|
||||
|
||||
CAPABILITY_REGISTRY["mcp_servers"] = MCP_CAPABILITIES
|
||||
CAPABILITY_REGISTRY["terraform_resources"] = TERRAFORM_RESOURCES
|
||||
CAPABILITY_REGISTRY["gitops_tools"] = GITOPS_TOOLS
|
||||
CAPABILITY_REGISTRY["security_framework"] = SECURITY_FRAMEWORK
|
||||
CAPABILITY_REGISTRY["operational_tools"] = OPERATIONAL_TOOLS
|
||||
|
||||
return CAPABILITY_REGISTRY
|
||||
|
||||
|
||||
def save_registry_formats():
|
||||
"""Save registry in multiple formats for different use cases."""
|
||||
|
||||
registry = generate_registry()
|
||||
|
||||
# JSON format (machine-readable)
|
||||
with open("capability_registry.json", "w") as f:
|
||||
json.dump(registry, f, indent=2)
|
||||
|
||||
# Markdown format (documentation)
|
||||
markdown_content = generate_markdown_doc(registry)
|
||||
with open("CAPABILITY_REGISTRY.md", "w") as f:
|
||||
f.write(markdown_content)
|
||||
|
||||
print("✅ Capability registry generated:")
|
||||
print(" - capability_registry.json (machine-readable)")
|
||||
print(" - CAPABILITY_REGISTRY.md (documentation)")
|
||||
|
||||
|
||||
def generate_markdown_doc(registry: Dict[str, Any]) -> str:
|
||||
"""Generate Markdown documentation from registry."""
|
||||
|
||||
md = f"""# Cloudflare Control Plane Capability Registry
|
||||
|
||||
Generated: {registry["metadata"]["generated_at"]}
|
||||
Version: {registry["metadata"]["version"]}
|
||||
|
||||
## MCP Servers
|
||||
|
||||
"""
|
||||
|
||||
for server_name, server_info in registry["mcp_servers"].items():
|
||||
md += f"### {server_name}\n"
|
||||
md += f"**Module**: `{server_info['module']}` \n"
|
||||
md += f"**Purpose**: {server_info['purpose']} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in server_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Terraform Resources\n\n"
|
||||
for resource_name, resource_info in registry["terraform_resources"].items():
|
||||
md += f"### {resource_name}\n"
|
||||
md += f"**Files**: {', '.join(resource_info['files'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in resource_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## GitOps Tools\n\n"
|
||||
for tool_name, tool_info in registry["gitops_tools"].items():
|
||||
md += f"### {tool_name}\n"
|
||||
md += f"**File**: {tool_info['file']} \n"
|
||||
md += f"**Purpose**: {tool_info['purpose']} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in tool_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Security Framework\n\n"
|
||||
for framework_name, framework_info in registry["security_framework"].items():
|
||||
md += f"### {framework_name}\n"
|
||||
md += f"**Components**: {', '.join(framework_info['components'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in framework_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "**Classification Levels**:\n"
|
||||
for level in framework_info["classification_levels"]:
|
||||
md += f"- {level}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Operational Tools\n\n"
|
||||
for tool_category, tool_info in registry["operational_tools"].items():
|
||||
md += f"### {tool_category}\n"
|
||||
if "services" in tool_info:
|
||||
md += f"**Services**: {', '.join(tool_info['services'])} \n\n"
|
||||
elif "suites" in tool_info:
|
||||
md += f"**Test Suites**: {', '.join(tool_info['suites'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in tool_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
return md
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
save_registry_formats()
|
||||
332
generate_capability_registry_v2.py
Normal file
332
generate_capability_registry_v2.py
Normal file
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cloudflare Control Plane Capability Registry Generator v2
|
||||
|
||||
Enhanced with exact MCP tool names, entrypoints, and operational details
|
||||
for audit-grade documentation and drift prevention.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Registry structure
|
||||
CAPABILITY_REGISTRY = {
|
||||
"metadata": {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"version": "1.0.1",
|
||||
"scope": "Cloudflare Control Plane",
|
||||
},
|
||||
"mcp_servers": {},
|
||||
"terraform_resources": {},
|
||||
"gitops_tools": {},
|
||||
"security_framework": {},
|
||||
"operational_tools": {},
|
||||
}
|
||||
|
||||
# MCP Server capabilities with exact tool names
|
||||
MCP_CAPABILITIES = {
|
||||
"cloudflare_safe": {
|
||||
"module": "cloudflare.mcp.cloudflare_safe",
|
||||
"entrypoint": "cloudflare.mcp.cloudflare_safe",
|
||||
"purpose": "Secure Cloudflare API operations",
|
||||
"tools": [
|
||||
"cf_snapshot (read/write token required)",
|
||||
"cf_refresh (write token required)",
|
||||
"cf_config_diff (read; requires snapshot_id)",
|
||||
"cf_export_config (read)",
|
||||
"cf_tunnel_status (read)",
|
||||
"cf_tunnel_ingress_summary (read)",
|
||||
"cf_access_policy_list (read)",
|
||||
],
|
||||
"auth_env": ["CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID"],
|
||||
"side_effects": "read-only unless token present; cf_refresh/cf_snapshot are mutating",
|
||||
"outputs": ["json", "terraform_hcl"],
|
||||
"capabilities": [
|
||||
"dns_record_management",
|
||||
"waf_rule_configuration",
|
||||
"tunnel_health_monitoring",
|
||||
"zone_analytics_query",
|
||||
"terraform_state_synchronization",
|
||||
],
|
||||
"security": {
|
||||
"token_redaction": True,
|
||||
"error_handling": True,
|
||||
"rate_limiting": True,
|
||||
},
|
||||
},
|
||||
"waf_intelligence": {
|
||||
"module": "cloudflare.mcp.waf_intelligence",
|
||||
"entrypoint": "cloudflare.mcp.waf_intelligence.mcp_server",
|
||||
"purpose": "WAF rule analysis and synthesis",
|
||||
"tools": [
|
||||
"waf_capabilities (read)",
|
||||
"waf_analyze (read)",
|
||||
"waf_assess (read)",
|
||||
"waf_generate_gitops_proposals (propose)",
|
||||
],
|
||||
"auth_env": [],
|
||||
"side_effects": "propose-only; generates GitOps proposals",
|
||||
"outputs": ["json", "terraform_hcl", "gitops_mr"],
|
||||
"capabilities": [
|
||||
"waf_config_analysis",
|
||||
"threat_intelligence_integration",
|
||||
"compliance_mapping",
|
||||
"rule_gap_identification",
|
||||
"terraform_ready_rule_generation",
|
||||
],
|
||||
"intelligence": {
|
||||
"ml_classification": True,
|
||||
"threat_intel": True,
|
||||
"compliance_frameworks": ["PCI-DSS 6.6", "OWASP-ASVS 13"],
|
||||
},
|
||||
},
|
||||
"oracle_answer": {
|
||||
"module": "cloudflare.mcp.oracle_answer",
|
||||
"entrypoint": "cloudflare.mcp.oracle_answer",
|
||||
"purpose": "Security decision support",
|
||||
"tools": ["oracle_answer (read)"],
|
||||
"auth_env": [],
|
||||
"side_effects": "read-only; security classification only",
|
||||
"outputs": ["json", "security_classification"],
|
||||
"capabilities": [
|
||||
"security_classification",
|
||||
"routing_decision_support",
|
||||
"threat_assessment",
|
||||
"pre_execution_screening",
|
||||
],
|
||||
"integration": {
|
||||
"layer0_framework": True,
|
||||
"shadow_classifier": True,
|
||||
"preboot_logging": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Terraform resources (from analysis)
|
||||
TERRAFORM_RESOURCES = {
|
||||
"dns_management": {
|
||||
"files": ["dns.tf"],
|
||||
"resources": ["cloudflare_record", "cloudflare_zone"],
|
||||
"capabilities": [
|
||||
"automated_dns_provisioning",
|
||||
"spf_dmarc_mx_configuration",
|
||||
"tunnel_based_routing",
|
||||
"proxied_record_management",
|
||||
],
|
||||
},
|
||||
"waf_security": {
|
||||
"files": ["waf.tf"],
|
||||
"resources": ["cloudflare_ruleset", "cloudflare_bot_management"],
|
||||
"capabilities": [
|
||||
"custom_waf_rules",
|
||||
"managed_ruleset_integration",
|
||||
"bot_management",
|
||||
"rate_limiting",
|
||||
"country_blocking",
|
||||
],
|
||||
},
|
||||
"tunnel_infrastructure": {
|
||||
"files": ["tunnels.tf"],
|
||||
"resources": ["cloudflare_tunnel", "cloudflare_tunnel_config"],
|
||||
"capabilities": [
|
||||
"multi_service_tunnel_routing",
|
||||
"ingress_rule_management",
|
||||
"health_monitoring",
|
||||
"credential_rotation",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# GitOps tools with operational details
|
||||
GITOPS_TOOLS = {
|
||||
"waf_rule_proposer": {
|
||||
"file": "gitops/waf_rule_proposer.py",
|
||||
"purpose": "Automated WAF rule generation",
|
||||
"side_effects": "creates GitLab merge requests",
|
||||
"outputs": ["terraform_hcl", "gitops_mr"],
|
||||
"capabilities": [
|
||||
"threat_intel_driven_rules",
|
||||
"gitlab_ci_integration",
|
||||
"automated_mr_creation",
|
||||
"compliance_mapping",
|
||||
],
|
||||
},
|
||||
"invariant_checker": {
|
||||
"file": "scripts/invariant_checker_py.py",
|
||||
"purpose": "Real-time state validation",
|
||||
"side_effects": "generates anomaly reports",
|
||||
"outputs": ["json", "anomaly_report"],
|
||||
"capabilities": [
|
||||
"dns_integrity_checks",
|
||||
"waf_compliance_validation",
|
||||
"tunnel_health_monitoring",
|
||||
"drift_detection",
|
||||
],
|
||||
},
|
||||
"drift_guardian": {
|
||||
"file": "scripts/drift_guardian_py.py",
|
||||
"purpose": "Automated remediation",
|
||||
"side_effects": "applies Terraform changes",
|
||||
"outputs": ["terraform_apply", "remediation_report"],
|
||||
"capabilities": [
|
||||
"state_reconciliation",
|
||||
"auto_remediation",
|
||||
"ops_notification",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Security framework
|
||||
SECURITY_FRAMEWORK = {
|
||||
"layer0": {
|
||||
"components": ["entrypoint.py", "shadow_classifier.py", "preboot_logger.py"],
|
||||
"capabilities": [
|
||||
"pre_execution_security_classification",
|
||||
"threat_assessment",
|
||||
"security_event_logging",
|
||||
"routing_decision_support",
|
||||
],
|
||||
"classification_levels": ["catastrophic", "forbidden", "ambiguous", "blessed"],
|
||||
}
|
||||
}
|
||||
|
||||
# Operational tools
|
||||
OPERATIONAL_TOOLS = {
|
||||
"systemd_services": {
|
||||
"services": ["autonomous-remediator", "drift-guardian", "tunnel-rotation"],
|
||||
"capabilities": [
|
||||
"continuous_monitoring",
|
||||
"automated_remediation",
|
||||
"scheduled_operations",
|
||||
],
|
||||
},
|
||||
"test_suites": {
|
||||
"suites": ["layer0_validation", "mcp_integration", "cloudflare_safe_ingress"],
|
||||
"capabilities": [
|
||||
"security_classification_testing",
|
||||
"mcp_server_validation",
|
||||
"api_integration_testing",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def generate_registry():
|
||||
"""Generate the complete capability registry."""
|
||||
|
||||
CAPABILITY_REGISTRY["mcp_servers"] = MCP_CAPABILITIES
|
||||
CAPABILITY_REGISTRY["terraform_resources"] = TERRAFORM_RESOURCES
|
||||
CAPABILITY_REGISTRY["gitops_tools"] = GITOPS_TOOLS
|
||||
CAPABILITY_REGISTRY["security_framework"] = SECURITY_FRAMEWORK
|
||||
CAPABILITY_REGISTRY["operational_tools"] = OPERATIONAL_TOOLS
|
||||
|
||||
return CAPABILITY_REGISTRY
|
||||
|
||||
|
||||
def save_registry_formats():
|
||||
"""Save registry in multiple formats for different use cases."""
|
||||
|
||||
registry = generate_registry()
|
||||
|
||||
# JSON format (machine-readable)
|
||||
with open("capability_registry_v2.json", "w") as f:
|
||||
json.dump(registry, f, indent=2)
|
||||
|
||||
# Markdown format (documentation)
|
||||
markdown_content = generate_markdown_doc(registry)
|
||||
with open("CAPABILITY_REGISTRY_V2.md", "w") as f:
|
||||
f.write(markdown_content)
|
||||
|
||||
print("✅ Enhanced capability registry generated:")
|
||||
print(" - capability_registry_v2.json (machine-readable)")
|
||||
print(" - CAPABILITY_REGISTRY_V2.md (documentation)")
|
||||
|
||||
|
||||
def generate_markdown_doc(registry: dict) -> str:
|
||||
"""Generate Markdown documentation from registry."""
|
||||
|
||||
md = f"""# Cloudflare Control Plane Capability Registry v2
|
||||
|
||||
Generated: {registry["metadata"]["generated_at"]}
|
||||
Version: {registry["metadata"]["version"]}
|
||||
|
||||
## MCP Servers
|
||||
|
||||
"""
|
||||
|
||||
for server_name, server_info in registry["mcp_servers"].items():
|
||||
md += f"### {server_name}\n"
|
||||
md += f"**Module**: `{server_info['module']}` \n"
|
||||
md += f"**Entrypoint**: `{server_info['entrypoint']}` \n"
|
||||
md += f"**Purpose**: {server_info['purpose']} \n\n"
|
||||
|
||||
md += "**Tools**:\n"
|
||||
for tool in server_info["tools"]:
|
||||
md += f"- {tool}\n"
|
||||
|
||||
md += f"\n**Auth/Env**: {', '.join(server_info['auth_env'])}\n"
|
||||
md += f"**Side Effects**: {server_info['side_effects']}\n"
|
||||
md += f"**Outputs**: {', '.join(server_info['outputs'])}\n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in server_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Terraform Resources\n\n"
|
||||
for resource_name, resource_info in registry["terraform_resources"].items():
|
||||
md += f"### {resource_name}\n"
|
||||
md += f"**Files**: {', '.join(resource_info['files'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in resource_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## GitOps Tools\n\n"
|
||||
for tool_name, tool_info in registry["gitops_tools"].items():
|
||||
md += f"### {tool_name}\n"
|
||||
md += f"**File**: {tool_info['file']} \n"
|
||||
md += f"**Purpose**: {tool_info['purpose']} \n"
|
||||
md += f"**Side Effects**: {tool_info['side_effects']} \n"
|
||||
md += f"**Outputs**: {', '.join(tool_info['outputs'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in tool_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Security Framework\n\n"
|
||||
for framework_name, framework_info in registry["security_framework"].items():
|
||||
md += f"### {framework_name}\n"
|
||||
md += f"**Components**: {', '.join(framework_info['components'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in framework_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "**Classification Levels**:\n"
|
||||
for level in framework_info["classification_levels"]:
|
||||
md += f"- {level}\n"
|
||||
md += "\n"
|
||||
|
||||
md += "## Operational Tools\n\n"
|
||||
for tool_category, tool_info in registry["operational_tools"].items():
|
||||
md += f"### {tool_category}\n"
|
||||
if "services" in tool_info:
|
||||
md += f"**Services**: {', '.join(tool_info['services'])} \n\n"
|
||||
elif "suites" in tool_info:
|
||||
md += f"**Test Suites**: {', '.join(tool_info['suites'])} \n\n"
|
||||
|
||||
md += "**Capabilities**:\n"
|
||||
for cap in tool_info["capabilities"]:
|
||||
md += f"- {cap}\n"
|
||||
md += "\n"
|
||||
|
||||
return md
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
save_registry_formats()
|
||||
392
layer0/learn.py
Normal file
392
layer0/learn.py
Normal file
@@ -0,0 +1,392 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .pattern_store import (
|
||||
normalize_query_for_matching,
|
||||
pattern_dict,
|
||||
write_pattern_snapshot,
|
||||
)
|
||||
|
||||
THIS_FILE = Path(__file__).resolve()
|
||||
LAYER0_DIR = THIS_FILE.parent
|
||||
REPO_ROOT = LAYER0_DIR.parent.parent
|
||||
|
||||
|
||||
def _utc_now_iso_z() -> str:
|
||||
return (
|
||||
datetime.now(timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def _default_db_path() -> Path:
|
||||
for key in ("LEDGER_DB_PATH", "VAULTMESH_LEDGER_DB"):
|
||||
v = (os.environ.get(key) or "").strip()
|
||||
if v:
|
||||
return Path(v).expanduser().resolve()
|
||||
return (REPO_ROOT / ".state" / "ledger.sqlite").resolve()
|
||||
|
||||
|
||||
def _default_candidate_path() -> Path:
|
||||
return (REPO_ROOT / ".state" / "layer0_patterns_candidate.json").resolve()
|
||||
|
||||
|
||||
def _read_jsonl(paths: Iterable[Path]) -> list[dict[str, Any]]:
|
||||
events: list[dict[str, Any]] = []
|
||||
for path in paths:
|
||||
if not path.exists():
|
||||
continue
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
events.append(obj)
|
||||
return events
|
||||
|
||||
|
||||
def _telemetry_actor(event: dict[str, Any]) -> str | None:
|
||||
v = event.get("actor") or event.get("user") or event.get("account")
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
meta = event.get("metadata")
|
||||
if isinstance(meta, dict):
|
||||
v2 = meta.get("actor") or meta.get("account")
|
||||
if isinstance(v2, str) and v2.strip():
|
||||
return v2.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _telemetry_trace_id(event: dict[str, Any]) -> str | None:
|
||||
for k in ("trace_id", "layer0_trace_id", "trace", "id"):
|
||||
v = event.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _telemetry_ts(event: dict[str, Any]) -> str | None:
|
||||
for k in ("timestamp", "ts", "time"):
|
||||
v = event.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _telemetry_query(event: dict[str, Any]) -> str:
|
||||
v = event.get("query") or event.get("prompt") or event.get("input")
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
meta = event.get("metadata")
|
||||
if isinstance(meta, dict) and isinstance(meta.get("query"), str):
|
||||
return str(meta.get("query"))
|
||||
return ""
|
||||
|
||||
|
||||
def _outcome(event: dict[str, Any]) -> str | None:
|
||||
v = event.get("outcome") or event.get("result") or event.get("status")
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _layer0_classification(event: dict[str, Any]) -> str | None:
|
||||
v = event.get("layer0_classification") or event.get("classification")
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _infer_target_from_event(
|
||||
event: dict[str, Any], *, include_relaxations: bool
|
||||
) -> tuple[str, str] | None:
|
||||
"""
|
||||
Returns (mode, classification) or None.
|
||||
|
||||
mode:
|
||||
- "escalate": adds/strengthens detection immediately
|
||||
- "relax": can reduce severity only after replay + explicit approval
|
||||
"""
|
||||
outcome = (_outcome(event) or "").lower()
|
||||
l0 = (_layer0_classification(event) or "").lower()
|
||||
|
||||
# Ground-truth blocked downstream: L0 should tighten.
|
||||
if outcome in {
|
||||
"blocked_by_guardrails",
|
||||
"blocked_by_policy",
|
||||
"blocked",
|
||||
"denied",
|
||||
} and l0 in {"blessed", "ambiguous"}:
|
||||
return ("escalate", "forbidden")
|
||||
|
||||
if (
|
||||
outcome in {"fail_closed", "catastrophic", "blocked_catastrophic"}
|
||||
and l0 != "catastrophic"
|
||||
):
|
||||
return ("escalate", "catastrophic")
|
||||
|
||||
# Preboot logs (already blocked) can still be used to learn more specific signatures.
|
||||
if not outcome and l0 in {"forbidden", "catastrophic"}:
|
||||
return ("escalate", l0)
|
||||
|
||||
# False positives: relax only after replay + approval.
|
||||
if include_relaxations and outcome in {"success", "ok"} and l0 in {"forbidden"}:
|
||||
return ("relax", "blessed")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _default_risk_score(classification: str) -> int:
|
||||
if classification == "catastrophic":
|
||||
return 5
|
||||
if classification == "forbidden":
|
||||
return 3
|
||||
if classification == "ambiguous":
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Bucket:
|
||||
traces: set[str]
|
||||
actors: set[str]
|
||||
last_seen: str | None
|
||||
|
||||
|
||||
def _ensure_ledger_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS migrations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS proof_artifacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
ts TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
kind TEXT NOT NULL,
|
||||
path TEXT,
|
||||
sha256_hex TEXT,
|
||||
blake3_hex TEXT,
|
||||
size_bytes INTEGER,
|
||||
meta_json TEXT,
|
||||
trace_id TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _log_artifact(
|
||||
*,
|
||||
kind: str,
|
||||
path: Path | None,
|
||||
meta: dict[str, Any],
|
||||
trace_id: str | None,
|
||||
db_path: Path,
|
||||
) -> str:
|
||||
try:
|
||||
from ledger.db import log_proof_artifact # type: ignore
|
||||
|
||||
return log_proof_artifact(
|
||||
kind=kind,
|
||||
path=path,
|
||||
meta=meta,
|
||||
trace_id=trace_id,
|
||||
db_path=db_path,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
artifact_id = str(uuid.uuid4())
|
||||
rel_path: str | None = None
|
||||
sha256_hex: str | None = None
|
||||
size_bytes: int | None = None
|
||||
|
||||
if path is not None:
|
||||
try:
|
||||
rel_path = str(path.resolve().relative_to(REPO_ROOT))
|
||||
except Exception:
|
||||
rel_path = str(path)
|
||||
if path.exists() and path.is_file():
|
||||
data = path.read_bytes()
|
||||
sha256_hex = hashlib.sha256(data).hexdigest()
|
||||
size_bytes = len(data)
|
||||
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
_ensure_ledger_schema(conn)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO proof_artifacts (
|
||||
id, ts, kind, path, sha256_hex, blake3_hex, size_bytes, meta_json, trace_id
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?);
|
||||
""",
|
||||
(
|
||||
artifact_id,
|
||||
_utc_now_iso_z(),
|
||||
kind,
|
||||
rel_path,
|
||||
sha256_hex,
|
||||
size_bytes,
|
||||
json.dumps(meta, ensure_ascii=False, sort_keys=True),
|
||||
trace_id,
|
||||
),
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return artifact_id
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Layer0: build candidate patterns from telemetry."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--telemetry-jsonl",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Path to telemetry JSONL (repeatable). Defaults include anomalies/preboot_shield.jsonl if present.",
|
||||
)
|
||||
parser.add_argument("--min-support", type=int, default=3)
|
||||
parser.add_argument("--min-actors", type=int, default=2)
|
||||
parser.add_argument("--max-tokens", type=int, default=8)
|
||||
parser.add_argument(
|
||||
"--include-relaxations",
|
||||
action="store_true",
|
||||
help="Generate relaxation candidates (still requires replay + explicit promotion).",
|
||||
)
|
||||
parser.add_argument("--out", type=str, default=str(_default_candidate_path()))
|
||||
parser.add_argument("--db", type=str, default=None)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
paths: list[Path] = []
|
||||
for p in args.telemetry_jsonl:
|
||||
if p:
|
||||
paths.append(Path(p).expanduser())
|
||||
|
||||
default_preboot = REPO_ROOT / "anomalies" / "preboot_shield.jsonl"
|
||||
if default_preboot.exists() and default_preboot not in paths:
|
||||
paths.append(default_preboot)
|
||||
|
||||
events = _read_jsonl(paths)
|
||||
|
||||
buckets: dict[tuple[str, str, tuple[str, ...]], _Bucket] = {}
|
||||
for ev in events:
|
||||
inferred = _infer_target_from_event(
|
||||
ev, include_relaxations=bool(args.include_relaxations)
|
||||
)
|
||||
if not inferred:
|
||||
continue
|
||||
mode, target = inferred
|
||||
|
||||
norm = normalize_query_for_matching(_telemetry_query(ev))
|
||||
tokens = norm.split()
|
||||
if len(tokens) < 2:
|
||||
continue
|
||||
if args.max_tokens and len(tokens) > args.max_tokens:
|
||||
tokens = tokens[: int(args.max_tokens)]
|
||||
|
||||
key = (mode, target, tuple(tokens))
|
||||
b = buckets.get(key)
|
||||
if b is None:
|
||||
b = _Bucket(traces=set(), actors=set(), last_seen=None)
|
||||
buckets[key] = b
|
||||
|
||||
trace = _telemetry_trace_id(ev)
|
||||
if trace:
|
||||
b.traces.add(trace)
|
||||
actor = _telemetry_actor(ev)
|
||||
if actor:
|
||||
b.actors.add(actor)
|
||||
ts = _telemetry_ts(ev)
|
||||
if ts and (b.last_seen is None or ts > b.last_seen):
|
||||
b.last_seen = ts
|
||||
|
||||
patterns: list[dict[str, Any]] = []
|
||||
for (mode, target, tokens), bucket in buckets.items():
|
||||
support = len(bucket.traces) if bucket.traces else 0
|
||||
actors = len(bucket.actors)
|
||||
if support < int(args.min_support):
|
||||
continue
|
||||
if actors and actors < int(args.min_actors):
|
||||
continue
|
||||
|
||||
patterns.append(
|
||||
pattern_dict(
|
||||
tokens_all=tokens,
|
||||
classification=target,
|
||||
reason="telemetry_learned",
|
||||
risk_score=_default_risk_score(target),
|
||||
flags=["telemetry_learned"],
|
||||
min_support=support,
|
||||
last_seen=bucket.last_seen,
|
||||
source={"support_traces": support, "support_actors": actors},
|
||||
mode=mode,
|
||||
pattern_id=str(uuid.uuid4()),
|
||||
)
|
||||
)
|
||||
|
||||
# Deterministic ordering: most severe, then most specific/support.
|
||||
severity_rank = {
|
||||
"blessed": 0,
|
||||
"ambiguous": 1,
|
||||
"forbidden": 2,
|
||||
"catastrophic": 3,
|
||||
}
|
||||
patterns.sort(
|
||||
key=lambda p: (
|
||||
severity_rank.get(p["classification"], 0),
|
||||
int(p.get("specificity_score") or 0),
|
||||
int(p.get("min_support") or 0),
|
||||
str(p.get("last_seen") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
out_path = Path(args.out).expanduser().resolve()
|
||||
write_pattern_snapshot(out_path, patterns)
|
||||
|
||||
db_path = Path(args.db).expanduser().resolve() if args.db else _default_db_path()
|
||||
artifact_id = _log_artifact(
|
||||
kind="shadow_pattern_candidate",
|
||||
path=out_path,
|
||||
meta={
|
||||
"patterns": len(patterns),
|
||||
"min_support": int(args.min_support),
|
||||
"min_actors": int(args.min_actors),
|
||||
"inputs": [str(p) for p in paths],
|
||||
},
|
||||
trace_id=None,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
print(f"Wrote {len(patterns)} candidate patterns to {out_path}")
|
||||
print(f"Logged artifact {artifact_id} to {db_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
331
layer0/pattern_store.py
Normal file
331
layer0/pattern_store.py
Normal file
@@ -0,0 +1,331 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
THIS_FILE = Path(__file__).resolve()
|
||||
LAYER0_DIR = THIS_FILE.parent
|
||||
REPO_ROOT = LAYER0_DIR.parent.parent
|
||||
|
||||
|
||||
_RE_URL = re.compile(r"\bhttps?://\S+\b", re.IGNORECASE)
|
||||
_RE_EMAIL = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
|
||||
_RE_IPV4 = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
|
||||
_RE_IPV6 = re.compile(r"\b(?:[0-9a-f]{0,4}:){2,}[0-9a-f]{0,4}\b", re.IGNORECASE)
|
||||
_RE_UUID = re.compile(
|
||||
r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_HEX_LONG = re.compile(r"\b[0-9a-f]{32,}\b", re.IGNORECASE)
|
||||
_RE_BASE64ISH = re.compile(r"\b[A-Za-z0-9+/]{28,}={0,2}\b")
|
||||
_RE_PATHISH = re.compile(r"(?:(?:\.\.?/)|/)[A-Za-z0-9._~/-]{2,}")
|
||||
_RE_NUMBER = re.compile(r"\b\d+\b")
|
||||
_RE_TOKEN = re.compile(r"[a-z][a-z_-]{1,31}", re.IGNORECASE)
|
||||
|
||||
|
||||
SAFE_VOCAB = {
|
||||
# Governance / safety verbs
|
||||
"disable",
|
||||
"override",
|
||||
"bypass",
|
||||
"skip",
|
||||
"ignore",
|
||||
"evade",
|
||||
"break",
|
||||
"force",
|
||||
"apply",
|
||||
"deploy",
|
||||
"destroy",
|
||||
"delete",
|
||||
"drop",
|
||||
"remove",
|
||||
"exfiltrate",
|
||||
# Critical nouns / domains
|
||||
"guardrails",
|
||||
"permissions",
|
||||
"governance",
|
||||
"git",
|
||||
"gitops",
|
||||
"dashboard",
|
||||
"manual",
|
||||
"prod",
|
||||
"production",
|
||||
"staging",
|
||||
"terraform",
|
||||
"waf",
|
||||
"dns",
|
||||
"tunnel",
|
||||
"access",
|
||||
"token",
|
||||
"secret",
|
||||
"key",
|
||||
"credential",
|
||||
"admin",
|
||||
"root",
|
||||
# Phrases often seen in L0 rules (tokenized)
|
||||
"self",
|
||||
"modifying",
|
||||
"directly",
|
||||
}
|
||||
|
||||
|
||||
def _utc_now_iso_z() -> str:
|
||||
return (
|
||||
datetime.now(timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def normalize_query_for_matching(query: str) -> str:
|
||||
"""
|
||||
Produce a low-leakage normalized string suitable for storing and matching.
|
||||
|
||||
Invariants:
|
||||
- Never stores raw URLs, IPs, emails, long hex strings, base64ish blobs, UUIDs, or paths.
|
||||
- Numbers are stripped to <NUM>.
|
||||
- Only safe vocabulary tokens are preserved; other words are dropped.
|
||||
"""
|
||||
q = (query or "").lower().strip()
|
||||
if not q:
|
||||
return ""
|
||||
|
||||
# Keep placeholders lowercase to make matching stable across sources.
|
||||
q = _RE_URL.sub("<url>", q)
|
||||
q = _RE_EMAIL.sub("<email>", q)
|
||||
q = _RE_IPV4.sub("<ip>", q)
|
||||
q = _RE_IPV6.sub("<ip>", q)
|
||||
q = _RE_UUID.sub("<uuid>", q)
|
||||
q = _RE_PATHISH.sub("<path>", q)
|
||||
q = _RE_HEX_LONG.sub("<hex>", q)
|
||||
q = _RE_BASE64ISH.sub("<b64>", q)
|
||||
q = _RE_NUMBER.sub("<num>", q)
|
||||
|
||||
# Tokenize; keep placeholders and a tight safe vocabulary.
|
||||
tokens: list[str] = []
|
||||
for raw in re.split(r"[^a-z0-9_<>\-_/]+", q):
|
||||
t = raw.strip()
|
||||
if not t:
|
||||
continue
|
||||
if t.startswith("<") and t.endswith(">"):
|
||||
tokens.append(t)
|
||||
continue
|
||||
if _RE_TOKEN.fullmatch(t) and t in SAFE_VOCAB:
|
||||
tokens.append(t)
|
||||
|
||||
# De-dupe while preserving order.
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for t in tokens:
|
||||
if t in seen:
|
||||
continue
|
||||
seen.add(t)
|
||||
out.append(t)
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
def normalized_tokens(query: str) -> list[str]:
|
||||
s = normalize_query_for_matching(query)
|
||||
return s.split() if s else []
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LearnedPattern:
|
||||
pattern_id: str
|
||||
tokens_all: tuple[str, ...]
|
||||
classification: str
|
||||
reason: str | None
|
||||
risk_score: int
|
||||
flags: tuple[str, ...]
|
||||
specificity_score: int
|
||||
min_support: int
|
||||
last_seen: str | None
|
||||
source: dict[str, Any] | None
|
||||
mode: str # "escalate" | "relax"
|
||||
|
||||
def matches(self, normalized_query: str) -> bool:
|
||||
if not normalized_query:
|
||||
return False
|
||||
hay = set(normalized_query.split())
|
||||
return all(t in hay for t in self.tokens_all)
|
||||
|
||||
|
||||
def _default_active_path() -> Path:
|
||||
configured = os.environ.get("LAYER0_ACTIVE_PATTERNS_PATH")
|
||||
if configured:
|
||||
return Path(configured).expanduser().resolve()
|
||||
return (REPO_ROOT / ".state" / "layer0_patterns_active.json").resolve()
|
||||
|
||||
|
||||
class PatternStore:
|
||||
"""
|
||||
Read-only active pattern snapshot.
|
||||
|
||||
This is intentionally immutable during request handling; mutations happen in
|
||||
offline jobs (learn/replay) that write a new snapshot and log an artifact.
|
||||
"""
|
||||
|
||||
def __init__(self, active_path: Path | None = None):
|
||||
self._active_path = active_path or _default_active_path()
|
||||
self._active: list[LearnedPattern] = []
|
||||
self._loaded = False
|
||||
|
||||
@property
|
||||
def active_path(self) -> Path:
|
||||
return self._active_path
|
||||
|
||||
def load(self) -> None:
|
||||
if self._loaded:
|
||||
return
|
||||
self._loaded = True
|
||||
self._active = self._load_patterns_file(self._active_path)
|
||||
|
||||
def patterns(self) -> list[LearnedPattern]:
|
||||
self.load()
|
||||
return list(self._active)
|
||||
|
||||
def match_ordered(self, normalized_query: str) -> list[LearnedPattern]:
|
||||
self.load()
|
||||
matched = [p for p in self._active if p.matches(normalized_query)]
|
||||
severity_rank = {
|
||||
"blessed": 0,
|
||||
"ambiguous": 1,
|
||||
"forbidden": 2,
|
||||
"catastrophic": 3,
|
||||
}
|
||||
matched.sort(
|
||||
key=lambda p: (
|
||||
severity_rank.get(p.classification, 0),
|
||||
p.specificity_score,
|
||||
p.min_support,
|
||||
p.last_seen or "",
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return matched
|
||||
|
||||
@staticmethod
|
||||
def _load_patterns_file(path: Path) -> list[LearnedPattern]:
|
||||
if not path.exists():
|
||||
return []
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
items = data.get("patterns") if isinstance(data, dict) else data
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
|
||||
patterns: list[LearnedPattern] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
tokens = item.get("tokens_all") or item.get("tokens") or []
|
||||
if not isinstance(tokens, list) or not tokens:
|
||||
continue
|
||||
tokens_norm = tuple(
|
||||
t.lower() if isinstance(t, str) else ""
|
||||
for t in tokens
|
||||
if isinstance(t, str)
|
||||
and t
|
||||
and (t.startswith("<") or t.lower() in SAFE_VOCAB)
|
||||
)
|
||||
if not tokens_norm:
|
||||
continue
|
||||
|
||||
classification = item.get("classification")
|
||||
if classification not in {
|
||||
"blessed",
|
||||
"ambiguous",
|
||||
"forbidden",
|
||||
"catastrophic",
|
||||
}:
|
||||
continue
|
||||
|
||||
flags = item.get("flags") or []
|
||||
if not isinstance(flags, list):
|
||||
flags = []
|
||||
|
||||
mode = item.get("mode") or "escalate"
|
||||
if mode not in {"escalate", "relax"}:
|
||||
mode = "escalate"
|
||||
|
||||
min_support = int(item.get("min_support") or item.get("support") or 0)
|
||||
specificity = int(item.get("specificity_score") or len(tokens_norm))
|
||||
risk_score = int(item.get("risk_score") or 0)
|
||||
|
||||
patterns.append(
|
||||
LearnedPattern(
|
||||
pattern_id=str(item.get("pattern_id") or item.get("id") or ""),
|
||||
tokens_all=tokens_norm,
|
||||
classification=classification,
|
||||
reason=item.get("reason"),
|
||||
risk_score=risk_score,
|
||||
flags=tuple(str(f) for f in flags if isinstance(f, str)),
|
||||
specificity_score=specificity,
|
||||
min_support=min_support,
|
||||
last_seen=item.get("last_seen"),
|
||||
source=item.get("source")
|
||||
if isinstance(item.get("source"), dict)
|
||||
else None,
|
||||
mode=mode,
|
||||
)
|
||||
)
|
||||
|
||||
severity_rank = {
|
||||
"blessed": 0,
|
||||
"ambiguous": 1,
|
||||
"forbidden": 2,
|
||||
"catastrophic": 3,
|
||||
}
|
||||
patterns.sort(
|
||||
key=lambda p: (
|
||||
severity_rank.get(p.classification, 0),
|
||||
p.specificity_score,
|
||||
p.min_support,
|
||||
p.last_seen or "",
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return patterns
|
||||
|
||||
|
||||
def pattern_dict(
|
||||
*,
|
||||
tokens_all: Sequence[str],
|
||||
classification: str,
|
||||
reason: str | None,
|
||||
risk_score: int,
|
||||
flags: Sequence[str],
|
||||
min_support: int,
|
||||
last_seen: str | None = None,
|
||||
source: dict[str, Any] | None = None,
|
||||
mode: str = "escalate",
|
||||
pattern_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
tokens = [t for t in tokens_all if isinstance(t, str) and t]
|
||||
return {
|
||||
"pattern_id": pattern_id or "",
|
||||
"tokens_all": tokens,
|
||||
"classification": classification,
|
||||
"reason": reason,
|
||||
"risk_score": int(risk_score),
|
||||
"flags": list(flags),
|
||||
"specificity_score": int(len(tokens)),
|
||||
"min_support": int(min_support),
|
||||
"last_seen": last_seen or _utc_now_iso_z(),
|
||||
"source": source or {},
|
||||
"mode": mode,
|
||||
}
|
||||
|
||||
|
||||
def write_pattern_snapshot(path: Path, patterns: Iterable[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"generated_at": _utc_now_iso_z(), "patterns": list(patterns)}
|
||||
path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
@@ -1,22 +1,134 @@
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .shadow_classifier import ShadowEvalResult, Classification
|
||||
from .pattern_store import normalize_query_for_matching
|
||||
from .shadow_classifier import Classification, ShadowEvalResult
|
||||
|
||||
|
||||
class PrebootLogger:
|
||||
LOG_PATH = "anomalies/preboot_shield.jsonl"
|
||||
|
||||
@staticmethod
|
||||
def _ledger_db_path() -> str | None:
|
||||
return os.getenv("VAULTMESH_LEDGER_DB") or os.getenv("LEDGER_DB_PATH")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_for_shadow_receipt(query: str) -> str:
|
||||
"""
|
||||
Poison-resistant normalizer for ShadowReceipt emission.
|
||||
|
||||
Goals:
|
||||
- Normalize casing/whitespace
|
||||
- Replace common secret/identifier carriers with placeholders
|
||||
- Keep output stable and compact
|
||||
"""
|
||||
s = (query or "").lower().strip()
|
||||
s = re.sub(r"\s+", " ", s)
|
||||
s = re.sub(r"\bhttps?://\S+\b", "<URL>", s)
|
||||
s = re.sub(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", "<IP>", s)
|
||||
s = re.sub(
|
||||
r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b",
|
||||
"<HEX>",
|
||||
s,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
s = re.sub(r"(?:(?:\.\.?/)|/|~\/)[A-Za-z0-9._~/-]{2,}", "<PATH>", s)
|
||||
s = re.sub(r"\b[0-9a-f]{16,}\b", "<HEX>", s, flags=re.IGNORECASE)
|
||||
s = re.sub(r"\b\d+\b", "<N>", s)
|
||||
return s.strip()
|
||||
|
||||
@staticmethod
|
||||
def _sha256_hex(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _try_emit_shadow_receipt(
|
||||
*,
|
||||
query: str,
|
||||
classification: str,
|
||||
reason: str | None,
|
||||
flags: list[str],
|
||||
trace_id: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
Best-effort ShadowReceipt emission into the local-first SQLite ledger.
|
||||
|
||||
Hard constraints:
|
||||
- No dependency on vaultmesh-orgine-mobile code
|
||||
- Fail silently on any error (Layer 0 must never crash)
|
||||
"""
|
||||
db_path = PrebootLogger._ledger_db_path()
|
||||
if not db_path:
|
||||
return
|
||||
|
||||
try:
|
||||
norm = PrebootLogger._normalize_for_shadow_receipt(query)
|
||||
cf_hash = PrebootLogger._sha256_hex(norm)
|
||||
|
||||
placeholders: list[str] = []
|
||||
for p in ("<URL>", "<IP>", "<PATH>", "<HEX>", "<N>"):
|
||||
if p in norm:
|
||||
placeholders.append(p)
|
||||
|
||||
meta = {
|
||||
"ts_utc": datetime.datetime.now(datetime.timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
"classification": classification,
|
||||
"reason": reason,
|
||||
"flags": (flags or [])[:64],
|
||||
"normalized_query_features": {
|
||||
"placeholders": placeholders,
|
||||
"length": len(norm),
|
||||
},
|
||||
}
|
||||
|
||||
conn = sqlite3.connect(db_path, timeout=0.25)
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys=ON;")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO shadow_receipts (
|
||||
id, horizon_id, counterfactual_hash, entropy_delta,
|
||||
reason_unrealized, observer_signature, trace_id, meta_json
|
||||
)
|
||||
VALUES (?, ?, ?, NULL, ?, NULL, ?, ?);
|
||||
""",
|
||||
(
|
||||
PrebootLogger._sha256_hex(
|
||||
meta["ts_utc"] + "|" + (trace_id or "") + "|" + cf_hash
|
||||
),
|
||||
"layer0_block",
|
||||
cf_hash,
|
||||
"layer0_block",
|
||||
trace_id,
|
||||
json.dumps(meta, separators=(",", ":"), ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def log(event: ShadowEvalResult, query: str, reason_override: Optional[str] = None):
|
||||
if event.classification not in (Classification.CATASTROPHIC, Classification.FORBIDDEN):
|
||||
if event.classification not in (
|
||||
Classification.CATASTROPHIC,
|
||||
Classification.FORBIDDEN,
|
||||
):
|
||||
return # Only violations get logged
|
||||
|
||||
record = {
|
||||
"timestamp": datetime.datetime.utcnow().isoformat() + "Z",
|
||||
"query": query,
|
||||
# Store a normalized, low-leakage representation (never raw strings).
|
||||
"query": normalize_query_for_matching(query),
|
||||
"classification": event.classification.value,
|
||||
"reason": reason_override or event.reason,
|
||||
"trace_id": event.trace_id,
|
||||
@@ -31,3 +143,11 @@ class PrebootLogger:
|
||||
|
||||
with open(PrebootLogger.LOG_PATH, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
|
||||
PrebootLogger._try_emit_shadow_receipt(
|
||||
query=query,
|
||||
classification=event.classification.value,
|
||||
reason=reason_override or event.reason,
|
||||
flags=event.flags,
|
||||
trace_id=event.trace_id,
|
||||
)
|
||||
|
||||
443
layer0/replay.py
Normal file
443
layer0/replay.py
Normal file
@@ -0,0 +1,443 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .pattern_store import PatternStore, write_pattern_snapshot
|
||||
from .shadow_classifier import Classification, ShadowClassifier
|
||||
|
||||
THIS_FILE = Path(__file__).resolve()
|
||||
LAYER0_DIR = THIS_FILE.parent
|
||||
REPO_ROOT = LAYER0_DIR.parent.parent
|
||||
|
||||
|
||||
def _utc_now_iso_z() -> str:
|
||||
return (
|
||||
datetime.now(timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
|
||||
def _default_db_path() -> Path:
|
||||
for key in ("LEDGER_DB_PATH", "VAULTMESH_LEDGER_DB"):
|
||||
v = (os.environ.get(key) or "").strip()
|
||||
if v:
|
||||
return Path(v).expanduser().resolve()
|
||||
return (REPO_ROOT / ".state" / "ledger.sqlite").resolve()
|
||||
|
||||
|
||||
def _read_jsonl(paths: Iterable[Path], *, limit: int | None) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for path in paths:
|
||||
if not path.exists():
|
||||
continue
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
rows.append(obj)
|
||||
|
||||
if limit is not None and limit > 0 and len(rows) > limit:
|
||||
return rows[-limit:]
|
||||
return rows
|
||||
|
||||
|
||||
def _telemetry_query(event: dict[str, Any]) -> str:
|
||||
v = event.get("query") or event.get("prompt") or event.get("input")
|
||||
return v if isinstance(v, str) else ""
|
||||
|
||||
|
||||
def _outcome(event: dict[str, Any]) -> str | None:
|
||||
v = event.get("outcome") or event.get("result") or event.get("status")
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _ground_truth(event: dict[str, Any]) -> Classification | None:
|
||||
outcome = (_outcome(event) or "").lower()
|
||||
if outcome in {"success", "ok"}:
|
||||
return Classification.BLESSED
|
||||
if outcome in {"blocked_by_guardrails", "blocked_by_policy", "blocked", "denied"}:
|
||||
return Classification.FORBIDDEN
|
||||
if outcome in {"fail_closed", "catastrophic", "blocked_catastrophic"}:
|
||||
return Classification.CATASTROPHIC
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_ledger_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS migrations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS proof_artifacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
ts TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
kind TEXT NOT NULL,
|
||||
path TEXT,
|
||||
sha256_hex TEXT,
|
||||
blake3_hex TEXT,
|
||||
size_bytes INTEGER,
|
||||
meta_json TEXT,
|
||||
trace_id TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _log_artifact(
|
||||
*,
|
||||
kind: str,
|
||||
path: Path | None,
|
||||
meta: dict[str, Any],
|
||||
trace_id: str | None,
|
||||
db_path: Path,
|
||||
) -> str:
|
||||
try:
|
||||
from ledger.db import log_proof_artifact # type: ignore
|
||||
|
||||
return log_proof_artifact(
|
||||
kind=kind,
|
||||
path=path,
|
||||
meta=meta,
|
||||
trace_id=trace_id,
|
||||
db_path=db_path,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
artifact_id = str(uuid.uuid4())
|
||||
rel_path: str | None = None
|
||||
sha256_hex: str | None = None
|
||||
size_bytes: int | None = None
|
||||
|
||||
if path is not None:
|
||||
try:
|
||||
rel_path = str(path.resolve().relative_to(REPO_ROOT))
|
||||
except Exception:
|
||||
rel_path = str(path)
|
||||
if path.exists() and path.is_file():
|
||||
data = path.read_bytes()
|
||||
sha256_hex = hashlib.sha256(data).hexdigest()
|
||||
size_bytes = len(data)
|
||||
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
_ensure_ledger_schema(conn)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO proof_artifacts (
|
||||
id, ts, kind, path, sha256_hex, blake3_hex, size_bytes, meta_json, trace_id
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?);
|
||||
""",
|
||||
(
|
||||
artifact_id,
|
||||
_utc_now_iso_z(),
|
||||
kind,
|
||||
rel_path,
|
||||
sha256_hex,
|
||||
size_bytes,
|
||||
json.dumps(meta, ensure_ascii=False, sort_keys=True),
|
||||
trace_id,
|
||||
),
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return artifact_id
|
||||
|
||||
|
||||
def _load_patterns_file(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
items = data.get("patterns") if isinstance(data, dict) else data
|
||||
return items if isinstance(items, list) else []
|
||||
|
||||
|
||||
def _merge_patterns(
|
||||
active: list[dict[str, Any]], extra: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Candidate patterns win on identical (mode, tokens_all, classification).
|
||||
"""
|
||||
|
||||
def key(p: dict[str, Any]) -> tuple[str, tuple[str, ...], str]:
|
||||
mode = str(p.get("mode") or "escalate")
|
||||
cls = str(p.get("classification") or "")
|
||||
tokens = p.get("tokens_all") or p.get("tokens") or []
|
||||
if not isinstance(tokens, list):
|
||||
tokens = []
|
||||
return (mode, tuple(str(t).lower() for t in tokens), cls)
|
||||
|
||||
merged: dict[tuple[str, tuple[str, ...], str], dict[str, Any]] = {}
|
||||
for p in active:
|
||||
if isinstance(p, dict):
|
||||
merged[key(p)] = p
|
||||
for p in extra:
|
||||
if isinstance(p, dict):
|
||||
merged[key(p)] = p
|
||||
return list(merged.values())
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReplayMetrics:
|
||||
total: int
|
||||
baseline_false_pos: int
|
||||
baseline_false_neg: int
|
||||
candidate_false_pos: int
|
||||
candidate_false_neg: int
|
||||
catastrophic_boundary_unchanged: bool
|
||||
|
||||
|
||||
def _is_false_positive(pred: Classification, truth: Classification) -> bool:
|
||||
return truth == Classification.BLESSED and pred in {
|
||||
Classification.FORBIDDEN,
|
||||
Classification.CATASTROPHIC,
|
||||
}
|
||||
|
||||
|
||||
def _is_false_negative(pred: Classification, truth: Classification) -> bool:
|
||||
return truth in {
|
||||
Classification.FORBIDDEN,
|
||||
Classification.CATASTROPHIC,
|
||||
} and pred in {
|
||||
Classification.BLESSED,
|
||||
Classification.AMBIGUOUS,
|
||||
}
|
||||
|
||||
|
||||
def _compute_metrics(
|
||||
events: list[dict[str, Any]],
|
||||
baseline: ShadowClassifier,
|
||||
candidate: ShadowClassifier,
|
||||
) -> ReplayMetrics:
|
||||
total = 0
|
||||
b_fp = b_fn = 0
|
||||
c_fp = c_fn = 0
|
||||
catastrophic_ok = True
|
||||
|
||||
for ev in events:
|
||||
truth = _ground_truth(ev)
|
||||
if truth is None:
|
||||
continue
|
||||
q = _telemetry_query(ev)
|
||||
total += 1
|
||||
|
||||
b = baseline.classify(q).classification
|
||||
c = candidate.classify(q).classification
|
||||
|
||||
if _is_false_positive(b, truth):
|
||||
b_fp += 1
|
||||
if _is_false_negative(b, truth):
|
||||
b_fn += 1
|
||||
if _is_false_positive(c, truth):
|
||||
c_fp += 1
|
||||
if _is_false_negative(c, truth):
|
||||
c_fn += 1
|
||||
|
||||
if b == Classification.CATASTROPHIC and c != Classification.CATASTROPHIC:
|
||||
catastrophic_ok = False
|
||||
|
||||
return ReplayMetrics(
|
||||
total=total,
|
||||
baseline_false_pos=b_fp,
|
||||
baseline_false_neg=b_fn,
|
||||
candidate_false_pos=c_fp,
|
||||
candidate_false_neg=c_fn,
|
||||
catastrophic_boundary_unchanged=catastrophic_ok,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Layer0: replay candidate patterns against recent telemetry."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--candidate",
|
||||
required=True,
|
||||
help="Candidate snapshot JSON (from layer0.learn).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--telemetry-jsonl",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Path to telemetry JSONL (repeatable). Must include outcome=success|blocked_by_guardrails|... for scoring.",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=2000)
|
||||
parser.add_argument(
|
||||
"--active",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Active patterns snapshot (defaults to .state).",
|
||||
)
|
||||
parser.add_argument("--db", type=str, default=None)
|
||||
parser.add_argument("--report-out", type=str, default=None)
|
||||
parser.add_argument(
|
||||
"--promote",
|
||||
action="store_true",
|
||||
help="If replay passes, write active snapshot update.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-relaxations",
|
||||
action="store_true",
|
||||
help="Allow promotion of relaxation-mode patterns (requires replay pass).",
|
||||
)
|
||||
parser.add_argument("--max-fp-increase", type=int, default=0)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
telemetry_paths = [Path(p).expanduser() for p in args.telemetry_jsonl if p]
|
||||
if not telemetry_paths:
|
||||
default_preboot = REPO_ROOT / "anomalies" / "preboot_shield.jsonl"
|
||||
if default_preboot.exists():
|
||||
telemetry_paths = [default_preboot]
|
||||
|
||||
events = _read_jsonl(telemetry_paths, limit=int(args.limit))
|
||||
|
||||
active_path = (
|
||||
Path(args.active).expanduser().resolve()
|
||||
if args.active
|
||||
else PatternStore().active_path
|
||||
)
|
||||
active_patterns = _load_patterns_file(active_path)
|
||||
candidate_path = Path(args.candidate).expanduser().resolve()
|
||||
candidate_patterns_all = _load_patterns_file(candidate_path)
|
||||
candidate_patterns = [
|
||||
p
|
||||
for p in candidate_patterns_all
|
||||
if isinstance(p, dict)
|
||||
and (args.allow_relaxations or str(p.get("mode") or "escalate") != "relax")
|
||||
]
|
||||
|
||||
baseline_classifier = ShadowClassifier(
|
||||
pattern_store=PatternStore(active_path=active_path)
|
||||
)
|
||||
|
||||
merged = _merge_patterns(active_patterns, candidate_patterns)
|
||||
merged_path = (
|
||||
REPO_ROOT / ".state" / "layer0_patterns_merged_replay.json"
|
||||
).resolve()
|
||||
write_pattern_snapshot(merged_path, merged)
|
||||
candidate_classifier = ShadowClassifier(
|
||||
pattern_store=PatternStore(active_path=merged_path)
|
||||
)
|
||||
|
||||
metrics = _compute_metrics(events, baseline_classifier, candidate_classifier)
|
||||
|
||||
passes = (
|
||||
metrics.catastrophic_boundary_unchanged
|
||||
and metrics.candidate_false_pos
|
||||
<= metrics.baseline_false_pos + int(args.max_fp_increase)
|
||||
and metrics.candidate_false_neg <= metrics.baseline_false_neg
|
||||
)
|
||||
|
||||
report = {
|
||||
"generated_at": _utc_now_iso_z(),
|
||||
"telemetry_inputs": [str(p) for p in telemetry_paths],
|
||||
"candidate_snapshot": str(candidate_path),
|
||||
"active_snapshot": str(active_path),
|
||||
"merged_snapshot": str(merged_path),
|
||||
"allow_relaxations": bool(args.allow_relaxations),
|
||||
"max_fp_increase": int(args.max_fp_increase),
|
||||
"metrics": {
|
||||
"total_scored": metrics.total,
|
||||
"baseline_false_positives": metrics.baseline_false_pos,
|
||||
"baseline_false_negatives": metrics.baseline_false_neg,
|
||||
"candidate_false_positives": metrics.candidate_false_pos,
|
||||
"candidate_false_negatives": metrics.candidate_false_neg,
|
||||
"catastrophic_boundary_unchanged": metrics.catastrophic_boundary_unchanged,
|
||||
},
|
||||
"passes": passes,
|
||||
"promotion": {
|
||||
"requested": bool(args.promote),
|
||||
"performed": False,
|
||||
"active_written_to": str(active_path),
|
||||
"patterns_added": len(candidate_patterns),
|
||||
},
|
||||
}
|
||||
|
||||
report_out = (
|
||||
Path(args.report_out).expanduser().resolve()
|
||||
if args.report_out
|
||||
else (REPO_ROOT / ".state" / "layer0_shadow_replay_report.json").resolve()
|
||||
)
|
||||
report_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_out.write_text(
|
||||
json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
db_path = Path(args.db).expanduser().resolve() if args.db else _default_db_path()
|
||||
report_artifact_id = _log_artifact(
|
||||
kind="shadow_replay_report",
|
||||
path=report_out,
|
||||
meta={
|
||||
"passes": passes,
|
||||
"total_scored": metrics.total,
|
||||
"baseline_fp": metrics.baseline_false_pos,
|
||||
"baseline_fn": metrics.baseline_false_neg,
|
||||
"candidate_fp": metrics.candidate_false_pos,
|
||||
"candidate_fn": metrics.candidate_false_neg,
|
||||
},
|
||||
trace_id=None,
|
||||
db_path=db_path,
|
||||
)
|
||||
|
||||
if args.promote and passes:
|
||||
# Promotion = merged active snapshot (existing + candidates), written atomically.
|
||||
tmp_path = active_path.with_suffix(active_path.suffix + ".tmp")
|
||||
write_pattern_snapshot(tmp_path, merged)
|
||||
tmp_path.replace(active_path)
|
||||
promo_artifact_id = _log_artifact(
|
||||
kind="shadow_pattern_promotion",
|
||||
path=active_path,
|
||||
meta={
|
||||
"added": len(candidate_patterns),
|
||||
"source_candidate": str(candidate_path),
|
||||
"merged_snapshot": str(merged_path),
|
||||
},
|
||||
trace_id=None,
|
||||
db_path=db_path,
|
||||
)
|
||||
report["promotion"]["performed"] = True
|
||||
report["promotion"]["artifact_id"] = promo_artifact_id
|
||||
report_out.write_text(
|
||||
json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"Replay report: {report_out} (passes={passes})")
|
||||
print(f"Logged artifact {report_artifact_id} to {db_path}")
|
||||
if args.promote:
|
||||
print(
|
||||
f"Promotion {'performed' if (args.promote and passes) else 'skipped'}; active={active_path}"
|
||||
)
|
||||
return 0 if passes else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
376
layer0/security_classifier.py
Normal file
376
layer0/security_classifier.py
Normal file
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced Security Classification Framework for Layer0
|
||||
Provides advanced classification capabilities for Cloudflare infrastructure operations
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional, Any
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
|
||||
|
||||
class SecurityLevel(str, Enum):
|
||||
"""Security classification levels"""
|
||||
|
||||
LOW_RISK = "low_risk"
|
||||
MEDIUM_RISK = "medium_risk"
|
||||
HIGH_RISK = "high_risk"
|
||||
CRITICAL_RISK = "critical_risk"
|
||||
|
||||
|
||||
class OperationType(str, Enum):
|
||||
"""Types of infrastructure operations"""
|
||||
|
||||
READ_ONLY = "read_only"
|
||||
CONFIGURATION_CHANGE = "configuration_change"
|
||||
INFRASTRUCTURE_MODIFICATION = "infrastructure_modification"
|
||||
SECURITY_MODIFICATION = "security_modification"
|
||||
ACCESS_CONTROL_CHANGE = "access_control_change"
|
||||
|
||||
|
||||
class ResourceType(str, Enum):
|
||||
"""Types of Cloudflare resources"""
|
||||
|
||||
DNS_RECORD = "dns_record"
|
||||
WAF_RULE = "waf_rule"
|
||||
ACCESS_RULE = "access_rule"
|
||||
TUNNEL = "tunnel"
|
||||
ZONE_SETTINGS = "zone_settings"
|
||||
ACCOUNT_SETTINGS = "account_settings"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SecurityClassification:
|
||||
"""Result of security classification"""
|
||||
|
||||
level: SecurityLevel
|
||||
operation_type: OperationType
|
||||
resource_type: ResourceType
|
||||
confidence: float # 0.0 to 1.0
|
||||
flags: List[str]
|
||||
rationale: str
|
||||
requires_approval: bool
|
||||
approval_threshold: Optional[str] = None
|
||||
|
||||
|
||||
class SecurityClassifier:
|
||||
"""
|
||||
Advanced security classifier for Cloudflare infrastructure operations
|
||||
Provides multi-dimensional risk assessment and classification
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Pattern definitions for different risk levels
|
||||
self.critical_patterns = [
|
||||
r"delete.*all",
|
||||
r"destroy.*infrastructure",
|
||||
r"disable.*waf",
|
||||
r"remove.*firewall",
|
||||
r"bypass.*security",
|
||||
r"expose.*credentials",
|
||||
r"terraform.*destroy",
|
||||
r"drop.*database",
|
||||
]
|
||||
|
||||
self.high_risk_patterns = [
|
||||
r"modify.*dns",
|
||||
r"change.*tunnel",
|
||||
r"update.*waf",
|
||||
r"create.*rule",
|
||||
r"modify.*access",
|
||||
r"terraform.*apply",
|
||||
]
|
||||
|
||||
self.medium_risk_patterns = [
|
||||
r"create.*record",
|
||||
r"update.*settings",
|
||||
r"configure.*zone",
|
||||
r"modify.*page",
|
||||
r"change.*cache",
|
||||
]
|
||||
|
||||
self.low_risk_patterns = [
|
||||
r"list.*records",
|
||||
r"get.*status",
|
||||
r"show.*config",
|
||||
r"read.*logs",
|
||||
r"monitor.*health",
|
||||
]
|
||||
|
||||
# Operation type patterns
|
||||
self.operation_patterns = {
|
||||
OperationType.READ_ONLY: [
|
||||
r"list",
|
||||
r"get",
|
||||
r"show",
|
||||
r"read",
|
||||
r"monitor",
|
||||
r"status",
|
||||
],
|
||||
OperationType.CONFIGURATION_CHANGE: [
|
||||
r"configure",
|
||||
r"update.*settings",
|
||||
r"change.*config",
|
||||
],
|
||||
OperationType.INFRASTRUCTURE_MODIFICATION: [
|
||||
r"create",
|
||||
r"modify",
|
||||
r"update",
|
||||
r"delete",
|
||||
r"destroy",
|
||||
],
|
||||
OperationType.SECURITY_MODIFICATION: [
|
||||
r"waf",
|
||||
r"firewall",
|
||||
r"security",
|
||||
r"block",
|
||||
r"allow",
|
||||
],
|
||||
OperationType.ACCESS_CONTROL_CHANGE: [
|
||||
r"access",
|
||||
r"permission",
|
||||
r"role",
|
||||
r"policy",
|
||||
],
|
||||
}
|
||||
|
||||
# Resource type patterns
|
||||
self.resource_patterns = {
|
||||
ResourceType.DNS_RECORD: [r"dns", r"record", r"domain", r"zone"],
|
||||
ResourceType.WAF_RULE: [r"waf", r"firewall", r"rule", r"security"],
|
||||
ResourceType.ACCESS_RULE: [r"access", r"policy", r"permission"],
|
||||
ResourceType.TUNNEL: [r"tunnel", r"connector", r"proxy"],
|
||||
ResourceType.ZONE_SETTINGS: [r"zone.*settings", r"domain.*config"],
|
||||
ResourceType.ACCOUNT_SETTINGS: [r"account.*settings", r"billing"],
|
||||
}
|
||||
|
||||
def classify_operation(
|
||||
self, operation_description: str, context: Optional[Dict[str, Any]] = None
|
||||
) -> SecurityClassification:
|
||||
"""
|
||||
Classify an infrastructure operation based on description and context
|
||||
"""
|
||||
description_lower = operation_description.lower()
|
||||
|
||||
# Determine security level
|
||||
security_level = self._determine_security_level(description_lower)
|
||||
|
||||
# Determine operation type
|
||||
operation_type = self._determine_operation_type(description_lower)
|
||||
|
||||
# Determine resource type
|
||||
resource_type = self._determine_resource_type(description_lower)
|
||||
|
||||
# Calculate confidence
|
||||
confidence = self._calculate_confidence(description_lower, security_level)
|
||||
|
||||
# Generate flags
|
||||
flags = self._generate_flags(description_lower, security_level, context)
|
||||
|
||||
# Generate rationale
|
||||
rationale = self._generate_rationale(
|
||||
security_level, operation_type, resource_type
|
||||
)
|
||||
|
||||
# Determine if approval is required
|
||||
requires_approval = security_level in [
|
||||
SecurityLevel.HIGH_RISK,
|
||||
SecurityLevel.CRITICAL_RISK,
|
||||
]
|
||||
approval_threshold = self._determine_approval_threshold(security_level)
|
||||
|
||||
return SecurityClassification(
|
||||
level=security_level,
|
||||
operation_type=operation_type,
|
||||
resource_type=resource_type,
|
||||
confidence=confidence,
|
||||
flags=flags,
|
||||
rationale=rationale,
|
||||
requires_approval=requires_approval,
|
||||
approval_threshold=approval_threshold,
|
||||
)
|
||||
|
||||
def _determine_security_level(self, description: str) -> SecurityLevel:
|
||||
"""Determine the security risk level"""
|
||||
for pattern in self.critical_patterns:
|
||||
if re.search(pattern, description):
|
||||
return SecurityLevel.CRITICAL_RISK
|
||||
|
||||
for pattern in self.high_risk_patterns:
|
||||
if re.search(pattern, description):
|
||||
return SecurityLevel.HIGH_RISK
|
||||
|
||||
for pattern in self.medium_risk_patterns:
|
||||
if re.search(pattern, description):
|
||||
return SecurityLevel.MEDIUM_RISK
|
||||
|
||||
for pattern in self.low_risk_patterns:
|
||||
if re.search(pattern, description):
|
||||
return SecurityLevel.LOW_RISK
|
||||
|
||||
# Default to medium risk for unknown operations
|
||||
return SecurityLevel.MEDIUM_RISK
|
||||
|
||||
def _determine_operation_type(self, description: str) -> OperationType:
|
||||
"""Determine the type of operation"""
|
||||
for op_type, patterns in self.operation_patterns.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, description):
|
||||
return op_type
|
||||
|
||||
# Default to infrastructure modification for safety
|
||||
return OperationType.INFRASTRUCTURE_MODIFICATION
|
||||
|
||||
def _determine_resource_type(self, description: str) -> ResourceType:
|
||||
"""Determine the type of resource being operated on"""
|
||||
for resource_type, patterns in self.resource_patterns.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, description):
|
||||
return resource_type
|
||||
|
||||
# Default to DNS records (most common)
|
||||
return ResourceType.DNS_RECORD
|
||||
|
||||
def _calculate_confidence(
|
||||
self, description: str, security_level: SecurityLevel
|
||||
) -> float:
|
||||
"""Calculate confidence score for classification"""
|
||||
base_confidence = 0.7
|
||||
|
||||
# Increase confidence for longer, more specific descriptions
|
||||
word_count = len(description.split())
|
||||
if word_count > 10:
|
||||
base_confidence += 0.2
|
||||
elif word_count > 5:
|
||||
base_confidence += 0.1
|
||||
|
||||
# Adjust based on security level
|
||||
if security_level == SecurityLevel.CRITICAL_RISK:
|
||||
base_confidence += 0.1 # Critical patterns are usually clear
|
||||
|
||||
return min(1.0, base_confidence)
|
||||
|
||||
def _generate_flags(
|
||||
self,
|
||||
description: str,
|
||||
security_level: SecurityLevel,
|
||||
context: Optional[Dict[str, Any]],
|
||||
) -> List[str]:
|
||||
"""Generate security flags for the operation"""
|
||||
flags = []
|
||||
|
||||
# Basic flags based on security level
|
||||
if security_level == SecurityLevel.CRITICAL_RISK:
|
||||
flags.extend(
|
||||
["critical_risk", "requires_emergency_approval", "multi_factor_auth"]
|
||||
)
|
||||
elif security_level == SecurityLevel.HIGH_RISK:
|
||||
flags.extend(["high_risk", "requires_senior_approval", "audit_trail"])
|
||||
elif security_level == SecurityLevel.MEDIUM_RISK:
|
||||
flags.extend(["medium_risk", "requires_standard_approval"])
|
||||
else:
|
||||
flags.extend(["low_risk", "auto_approved"])
|
||||
|
||||
# Context-based flags
|
||||
if context:
|
||||
environment = context.get("environment", "")
|
||||
if environment.lower() in ["prod", "production"]:
|
||||
flags.append("production_environment")
|
||||
|
||||
user_role = context.get("user_role", "")
|
||||
if user_role.lower() in ["admin", "root"]:
|
||||
flags.append("privileged_user")
|
||||
|
||||
# Pattern-based flags
|
||||
if re.search(r"delete|destroy|remove", description):
|
||||
flags.append("destructive_operation")
|
||||
|
||||
if re.search(r"waf|firewall|security", description):
|
||||
flags.append("security_related")
|
||||
|
||||
if re.search(r"dns|domain|zone", description):
|
||||
flags.append("dns_related")
|
||||
|
||||
return flags
|
||||
|
||||
def _generate_rationale(
|
||||
self,
|
||||
security_level: SecurityLevel,
|
||||
operation_type: OperationType,
|
||||
resource_type: ResourceType,
|
||||
) -> str:
|
||||
"""Generate rationale for the classification"""
|
||||
rationales = {
|
||||
SecurityLevel.CRITICAL_RISK: "Critical risk operation involving infrastructure destruction or security bypass",
|
||||
SecurityLevel.HIGH_RISK: "High risk operation modifying core infrastructure or security settings",
|
||||
SecurityLevel.MEDIUM_RISK: "Medium risk operation involving configuration changes",
|
||||
SecurityLevel.LOW_RISK: "Low risk read-only operation",
|
||||
}
|
||||
|
||||
base_rationale = rationales.get(
|
||||
security_level, "Standard infrastructure operation"
|
||||
)
|
||||
|
||||
# Add operation-specific details
|
||||
if operation_type == OperationType.INFRASTRUCTURE_MODIFICATION:
|
||||
base_rationale += " with infrastructure modification capabilities"
|
||||
elif operation_type == OperationType.SECURITY_MODIFICATION:
|
||||
base_rationale += " affecting security controls"
|
||||
|
||||
# Add resource-specific details
|
||||
if resource_type == ResourceType.DNS_RECORD:
|
||||
base_rationale += " on DNS infrastructure"
|
||||
elif resource_type == ResourceType.WAF_RULE:
|
||||
base_rationale += " on WAF security rules"
|
||||
|
||||
return base_rationale
|
||||
|
||||
def _determine_approval_threshold(
|
||||
self, security_level: SecurityLevel
|
||||
) -> Optional[str]:
|
||||
"""Determine the approval threshold required"""
|
||||
thresholds = {
|
||||
SecurityLevel.CRITICAL_RISK: "Emergency Change Advisory Board (ECAB)",
|
||||
SecurityLevel.HIGH_RISK: "Senior Infrastructure Engineer",
|
||||
SecurityLevel.MEDIUM_RISK: "Team Lead",
|
||||
SecurityLevel.LOW_RISK: None,
|
||||
}
|
||||
return thresholds.get(security_level)
|
||||
|
||||
|
||||
# Example usage and testing
|
||||
def main():
|
||||
"""Example usage of the security classifier"""
|
||||
classifier = SecurityClassifier()
|
||||
|
||||
# Test cases
|
||||
test_cases = [
|
||||
"Delete all DNS records for domain example.com",
|
||||
"Update WAF rule to allow traffic from China",
|
||||
"Create new DNS record for subdomain",
|
||||
"List all current tunnels and their status",
|
||||
"Modify zone settings to enable development mode",
|
||||
"Destroy all terraform infrastructure",
|
||||
]
|
||||
|
||||
print("🔐 Security Classification Framework Test")
|
||||
print("=" * 60)
|
||||
|
||||
for test_case in test_cases:
|
||||
classification = classifier.classify_operation(test_case)
|
||||
|
||||
print(f"\nOperation: {test_case}")
|
||||
print(f"Security Level: {classification.level.value}")
|
||||
print(f"Operation Type: {classification.operation_type.value}")
|
||||
print(f"Resource Type: {classification.resource_type.value}")
|
||||
print(f"Confidence: {classification.confidence:.2f}")
|
||||
print(f"Requires Approval: {classification.requires_approval}")
|
||||
if classification.approval_threshold:
|
||||
print(f"Approval Threshold: {classification.approval_threshold}")
|
||||
print(f"Flags: {', '.join(classification.flags)}")
|
||||
print(f"Rationale: {classification.rationale}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,8 @@
|
||||
from enum import Enum
|
||||
from typing import Optional, List
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from typing import Any, List, Mapping, Optional
|
||||
|
||||
from .pattern_store import PatternStore, normalize_query_for_matching
|
||||
|
||||
|
||||
class Classification(str, Enum):
|
||||
@@ -39,55 +41,136 @@ class ShadowClassifier:
|
||||
Minimal doctrinal classifier for Layer 0 (Shadow Eval).
|
||||
"""
|
||||
|
||||
def classify(self, query: str) -> ShadowEvalResult:
|
||||
def __init__(self, pattern_store: PatternStore | None = None):
|
||||
self._patterns = pattern_store or PatternStore()
|
||||
|
||||
def classify(
|
||||
self, query: str, *, context: Mapping[str, Any] | None = None
|
||||
) -> ShadowEvalResult:
|
||||
"""Return a doctrinal classification for the incoming query."""
|
||||
|
||||
q = query.lower().strip()
|
||||
q = (query or "").lower().strip()
|
||||
q_norm = normalize_query_for_matching(query or "")
|
||||
|
||||
# 1. Catastrophic (fail closed)
|
||||
if any(x in q for x in [
|
||||
"disable guardrails",
|
||||
"override agent permissions",
|
||||
"bypass governance",
|
||||
"self-modifying",
|
||||
]):
|
||||
return ShadowEvalResult(
|
||||
classification=Classification.CATASTROPHIC,
|
||||
reason="catastrophic_indicator",
|
||||
risk_score=5,
|
||||
flags=["permission_override", "guardrail_disable"],
|
||||
# 0. Catastrophic boundary (fail closed): never relaxed at runtime.
|
||||
if any(
|
||||
x in q
|
||||
for x in [
|
||||
"disable guardrails",
|
||||
"override agent permissions",
|
||||
"bypass governance",
|
||||
"self-modifying",
|
||||
]
|
||||
):
|
||||
return self._apply_context(
|
||||
ShadowEvalResult(
|
||||
classification=Classification.CATASTROPHIC,
|
||||
reason="catastrophic_indicator",
|
||||
risk_score=5,
|
||||
flags=["permission_override", "guardrail_disable"],
|
||||
),
|
||||
context,
|
||||
)
|
||||
|
||||
# 2. Forbidden (governance violation)
|
||||
if any(x in q for x in [
|
||||
"skip git",
|
||||
"apply directly",
|
||||
"dashboard",
|
||||
"manual change",
|
||||
]):
|
||||
return ShadowEvalResult(
|
||||
classification=Classification.FORBIDDEN,
|
||||
reason="governance_violation",
|
||||
risk_score=3,
|
||||
flags=["gitops_bypass"],
|
||||
# 1. Learned patterns (highest specificity/support first)
|
||||
learned = self._patterns.match_ordered(q_norm)
|
||||
if learned:
|
||||
p = learned[0]
|
||||
return self._apply_context(
|
||||
ShadowEvalResult(
|
||||
classification=Classification(p.classification),
|
||||
reason=p.reason or "telemetry_learned",
|
||||
risk_score=int(p.risk_score),
|
||||
flags=list(p.flags) + ["telemetry_learned"],
|
||||
),
|
||||
context,
|
||||
)
|
||||
|
||||
# 3. Ambiguous (needs clarification)
|
||||
if any(x in q for x in [
|
||||
"fix it",
|
||||
"change this",
|
||||
"update stuff",
|
||||
]) or len(q.split()) <= 2:
|
||||
return ShadowEvalResult(
|
||||
classification=Classification.AMBIGUOUS,
|
||||
reason="insufficient_context",
|
||||
risk_score=1,
|
||||
flags=["needs_clarification"],
|
||||
# 2. Static patterns
|
||||
|
||||
# 2a. Forbidden (governance violation)
|
||||
if any(
|
||||
x in q
|
||||
for x in [
|
||||
"skip git",
|
||||
"apply directly",
|
||||
"dashboard",
|
||||
"manual change",
|
||||
]
|
||||
):
|
||||
return self._apply_context(
|
||||
ShadowEvalResult(
|
||||
classification=Classification.FORBIDDEN,
|
||||
reason="governance_violation",
|
||||
risk_score=3,
|
||||
flags=["gitops_bypass"],
|
||||
),
|
||||
context,
|
||||
)
|
||||
|
||||
# 2b. Ambiguous (needs clarification)
|
||||
if (
|
||||
any(
|
||||
x in q
|
||||
for x in [
|
||||
"fix it",
|
||||
"change this",
|
||||
"update stuff",
|
||||
]
|
||||
)
|
||||
or len(q.split()) <= 2
|
||||
):
|
||||
return self._apply_context(
|
||||
ShadowEvalResult(
|
||||
classification=Classification.AMBIGUOUS,
|
||||
reason="insufficient_context",
|
||||
risk_score=1,
|
||||
flags=["needs_clarification"],
|
||||
),
|
||||
context,
|
||||
)
|
||||
|
||||
# 4. Blessed (valid + lawful)
|
||||
return ShadowEvalResult(
|
||||
classification=Classification.BLESSED,
|
||||
reason=None,
|
||||
risk_score=0,
|
||||
return self._apply_context(
|
||||
ShadowEvalResult(
|
||||
classification=Classification.BLESSED,
|
||||
reason=None,
|
||||
risk_score=0,
|
||||
),
|
||||
context,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_context(
|
||||
result: ShadowEvalResult, context: Mapping[str, Any] | None
|
||||
) -> ShadowEvalResult:
|
||||
if not context:
|
||||
return result
|
||||
|
||||
env = str(context.get("environment") or "").lower()
|
||||
realm = str(context.get("realm") or "").lower()
|
||||
capability = str(context.get("capability") or "").lower()
|
||||
role = str(context.get("actor_role") or context.get("role") or "").lower()
|
||||
|
||||
mult = 1.0
|
||||
if env in {"prod", "production"}:
|
||||
mult *= 2.0
|
||||
elif env in {"staging", "stage"}:
|
||||
mult *= 1.5
|
||||
elif env in {"dev", "development", "test"}:
|
||||
mult *= 1.0
|
||||
|
||||
if capability in {"destroy", "delete", "write"}:
|
||||
mult *= 1.5
|
||||
elif capability in {"read"}:
|
||||
mult *= 1.0
|
||||
|
||||
if role in {"admin", "root"}:
|
||||
mult *= 1.2
|
||||
|
||||
if realm in {"terraform", "gitops", "cloudflare"}:
|
||||
mult *= 1.1
|
||||
|
||||
weighted = int(round(result.risk_score * mult))
|
||||
result.risk_score = max(0, min(5, weighted))
|
||||
return result
|
||||
|
||||
@@ -3,4 +3,6 @@ MCP tools for the CLOUDFLARE workspace.
|
||||
|
||||
Currently:
|
||||
- oracle_answer: compliance / security oracle
|
||||
- cloudflare_safe: summary-first Cloudflare state + tunnel helpers
|
||||
- akash_docs: Akash docs fetch/search + SDL template helper
|
||||
"""
|
||||
|
||||
10
mcp/akash_docs/__init__.py
Normal file
10
mcp/akash_docs/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
Akash docs + deployment helpers exposed as an MCP server.
|
||||
|
||||
Tools:
|
||||
- akash_docs_list_routes: discover common docs routes from akash.network
|
||||
- akash_docs_fetch: fetch a docs page (prefers GitHub markdown, falls back to site HTML)
|
||||
- akash_docs_search: keyword search across discovered routes (cached)
|
||||
- akash_sdl_snippet: generate a minimal Akash SDL template
|
||||
"""
|
||||
|
||||
7
mcp/akash_docs/__main__.py
Normal file
7
mcp/akash_docs/__main__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
861
mcp/akash_docs/server.py
Normal file
861
mcp/akash_docs/server.py
Normal file
@@ -0,0 +1,861 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
AKASH_SITE_BASE = "https://akash.network"
|
||||
AKASH_DOCS_BASE = f"{AKASH_SITE_BASE}/docs"
|
||||
|
||||
AKASH_DOCS_GITHUB_OWNER = "akash-network"
|
||||
AKASH_DOCS_GITHUB_REPO = "website-revamp"
|
||||
AKASH_DOCS_GITHUB_REF_DEFAULT = "main"
|
||||
AKASH_DOCS_GITHUB_DOCS_ROOT = "src/content/Docs"
|
||||
|
||||
MAX_BYTES_DEFAULT = 32_000
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
# server.py -> akash_docs -> mcp -> cloudflare -> <repo root>
|
||||
return Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _max_bytes() -> int:
|
||||
raw = (os.getenv("VM_MCP_MAX_BYTES") or "").strip()
|
||||
if not raw:
|
||||
return MAX_BYTES_DEFAULT
|
||||
try:
|
||||
return max(4_096, int(raw))
|
||||
except ValueError:
|
||||
return MAX_BYTES_DEFAULT
|
||||
|
||||
|
||||
def _sha256_hex(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _http_get(url: str, *, timeout: int = 30) -> str:
|
||||
req = urllib.request.Request(
|
||||
url=url,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"User-Agent": "work-core-akamcp/0.1 (+https://akash.network)",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def _normalize_route(route_or_url: str) -> Tuple[str, str]:
|
||||
"""
|
||||
Returns (route, canonical_url).
|
||||
|
||||
route: "getting-started/what-is-akash" (no leading/trailing slashes)
|
||||
canonical_url: https://akash.network/docs/<route>
|
||||
"""
|
||||
raw = (route_or_url or "").strip()
|
||||
if not raw:
|
||||
return "", AKASH_DOCS_BASE + "/"
|
||||
|
||||
if raw.startswith("http://") or raw.startswith("https://"):
|
||||
parsed = urllib.parse.urlparse(raw)
|
||||
path = parsed.path or ""
|
||||
# Normalize to docs route if possible.
|
||||
if path in ("/docs", "/docs/"):
|
||||
return "", AKASH_DOCS_BASE + "/"
|
||||
if path.startswith("/docs/"):
|
||||
route = path[len("/docs/") :].strip("/")
|
||||
return route, f"{AKASH_DOCS_BASE}/{route}"
|
||||
return path.strip("/"), raw
|
||||
|
||||
# Accept "/docs/..." or "docs/..."
|
||||
route = raw.lstrip("/")
|
||||
if route in ("docs", "docs/"):
|
||||
return "", AKASH_DOCS_BASE + "/"
|
||||
if route.startswith("docs/"):
|
||||
route = route[len("docs/") :]
|
||||
route = route.strip("/")
|
||||
return route, f"{AKASH_DOCS_BASE}/{route}" if route else AKASH_DOCS_BASE + "/"
|
||||
|
||||
|
||||
def _strip_frontmatter(markdown: str) -> str:
|
||||
# Remove leading YAML frontmatter: ---\n...\n---\n
|
||||
if not markdown.startswith("---"):
|
||||
return markdown
|
||||
m = re.match(r"^---\s*\n.*?\n---\s*\n", markdown, flags=re.S)
|
||||
if not m:
|
||||
return markdown
|
||||
return markdown[m.end() :]
|
||||
|
||||
|
||||
def _github_candidates(route: str) -> List[str]:
|
||||
base = f"{AKASH_DOCS_GITHUB_DOCS_ROOT}/{route}".rstrip("/")
|
||||
candidates = [
|
||||
f"{base}/index.md",
|
||||
f"{base}/index.mdx",
|
||||
f"{base}.md",
|
||||
f"{base}.mdx",
|
||||
]
|
||||
# Handle root docs landing if route is empty.
|
||||
if not route:
|
||||
candidates = [
|
||||
f"{AKASH_DOCS_GITHUB_DOCS_ROOT}/index.md",
|
||||
f"{AKASH_DOCS_GITHUB_DOCS_ROOT}/index.mdx",
|
||||
]
|
||||
return candidates
|
||||
|
||||
|
||||
def _fetch_markdown_from_github(route: str, *, ref: str) -> Tuple[str, str, str]:
|
||||
"""
|
||||
Returns (markdown, raw_url, repo_path) or raises urllib.error.HTTPError.
|
||||
"""
|
||||
last_err: Optional[urllib.error.HTTPError] = None
|
||||
for repo_path in _github_candidates(route):
|
||||
raw_url = (
|
||||
f"https://raw.githubusercontent.com/{AKASH_DOCS_GITHUB_OWNER}/"
|
||||
f"{AKASH_DOCS_GITHUB_REPO}/{ref}/{repo_path}"
|
||||
)
|
||||
try:
|
||||
return _http_get(raw_url), raw_url, repo_path
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
last_err = e
|
||||
continue
|
||||
raise
|
||||
if last_err:
|
||||
raise last_err
|
||||
raise urllib.error.HTTPError(
|
||||
url="",
|
||||
code=404,
|
||||
msg="Not Found",
|
||||
hdrs=None,
|
||||
fp=None,
|
||||
)
|
||||
|
||||
|
||||
def _extract_article_html(page_html: str) -> str:
|
||||
m = re.search(r"<article\b[^>]*>(.*?)</article>", page_html, flags=re.S | re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.search(r"<main\b[^>]*>(.*?)</main>", page_html, flags=re.S | re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return page_html
|
||||
|
||||
|
||||
def _html_to_text(article_html: str) -> str:
|
||||
# Drop scripts/styles
|
||||
cleaned = re.sub(
|
||||
r"<(script|style)\b[^>]*>.*?</\1>", "", article_html, flags=re.S | re.I
|
||||
)
|
||||
|
||||
# Preserve code blocks a bit better (Astro uses <div class="ec-line"> for each line)
|
||||
def _pre_repl(match: re.Match[str]) -> str:
|
||||
pre = match.group(0)
|
||||
pre = re.sub(r"</div>\s*", "\n", pre, flags=re.I)
|
||||
pre = re.sub(r"<div\b[^>]*>", "", pre, flags=re.I)
|
||||
pre = re.sub(r"<br\s*/?>", "\n", pre, flags=re.I)
|
||||
pre = re.sub(r"<[^>]+>", "", pre)
|
||||
return "\n```\n" + _html_unescape(pre).strip() + "\n```\n"
|
||||
|
||||
cleaned = re.sub(r"<pre\b[^>]*>.*?</pre>", _pre_repl, cleaned, flags=re.S | re.I)
|
||||
|
||||
# Newlines for common block tags
|
||||
cleaned = re.sub(
|
||||
r"</(p|h1|h2|h3|h4|h5|h6|li|blockquote)>", "\n", cleaned, flags=re.I
|
||||
)
|
||||
cleaned = re.sub(r"<br\s*/?>", "\n", cleaned, flags=re.I)
|
||||
cleaned = re.sub(r"<hr\b[^>]*>", "\n---\n", cleaned, flags=re.I)
|
||||
|
||||
# Strip remaining tags
|
||||
cleaned = re.sub(r"<[^>]+>", "", cleaned)
|
||||
|
||||
text = _html_unescape(cleaned)
|
||||
lines = [ln.rstrip() for ln in text.splitlines()]
|
||||
# Collapse excessive blank lines
|
||||
out: List[str] = []
|
||||
blank = False
|
||||
for ln in lines:
|
||||
if ln.strip() == "":
|
||||
if blank:
|
||||
continue
|
||||
blank = True
|
||||
out.append("")
|
||||
continue
|
||||
blank = False
|
||||
out.append(ln.strip())
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def _html_unescape(text: str) -> str:
|
||||
# Avoid importing html module repeatedly; do it lazily.
|
||||
import html as _html # local import to keep global import list small
|
||||
|
||||
return _html.unescape(text)
|
||||
|
||||
|
||||
def _discover_routes_from_docs_index() -> List[str]:
|
||||
html = _http_get(AKASH_DOCS_BASE + "/")
|
||||
hrefs = set(re.findall(r'href=\"(/docs/[^\"#?]+)\"', html))
|
||||
routes: List[str] = []
|
||||
for href in sorted(hrefs):
|
||||
route, _url = _normalize_route(href)
|
||||
if route:
|
||||
routes.append(route)
|
||||
return routes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CachedDoc:
|
||||
cache_key: str
|
||||
fetched_at: str
|
||||
source: str
|
||||
route: str
|
||||
url: str
|
||||
ref: str
|
||||
content_path: str
|
||||
|
||||
|
||||
class DocStore:
|
||||
def __init__(self, root_dir: Path) -> None:
|
||||
self.root_dir = root_dir
|
||||
self.pages_dir = root_dir / "pages"
|
||||
self.index_path = root_dir / "index.json"
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._index: Dict[str, Dict[str, Any]] = {}
|
||||
if self.index_path.exists():
|
||||
try:
|
||||
self._index = json.loads(self.index_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
self._index = {}
|
||||
|
||||
def _write_index(self) -> None:
|
||||
tmp = self.index_path.with_suffix(".tmp")
|
||||
tmp.write_text(
|
||||
json.dumps(self._index, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
tmp.replace(self.index_path)
|
||||
|
||||
def get(self, cache_key: str) -> Optional[CachedDoc]:
|
||||
raw = self._index.get(cache_key)
|
||||
if not raw:
|
||||
return None
|
||||
path = Path(raw.get("content_path") or "")
|
||||
if not path.exists():
|
||||
return None
|
||||
return CachedDoc(
|
||||
cache_key=cache_key,
|
||||
fetched_at=str(raw.get("fetched_at") or ""),
|
||||
source=str(raw.get("source") or ""),
|
||||
route=str(raw.get("route") or ""),
|
||||
url=str(raw.get("url") or ""),
|
||||
ref=str(raw.get("ref") or ""),
|
||||
content_path=str(path),
|
||||
)
|
||||
|
||||
def save(
|
||||
self,
|
||||
*,
|
||||
cache_key: str,
|
||||
source: str,
|
||||
route: str,
|
||||
url: str,
|
||||
ref: str,
|
||||
content: str,
|
||||
) -> CachedDoc:
|
||||
content_hash = _sha256_hex(f"{source}:{ref}:{url}")[:20]
|
||||
path = self.pages_dir / f"{content_hash}.txt"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
entry = {
|
||||
"fetched_at": _utc_now_iso(),
|
||||
"source": source,
|
||||
"route": route,
|
||||
"url": url,
|
||||
"ref": ref,
|
||||
"content_path": str(path),
|
||||
}
|
||||
self._index[cache_key] = entry
|
||||
self._write_index()
|
||||
return self.get(cache_key) or CachedDoc(
|
||||
cache_key=cache_key,
|
||||
fetched_at=entry["fetched_at"],
|
||||
source=source,
|
||||
route=route,
|
||||
url=url,
|
||||
ref=ref,
|
||||
content_path=str(path),
|
||||
)
|
||||
|
||||
|
||||
def _default_state_dir() -> Path:
|
||||
return _repo_root() / "archive_runtime" / "akash_docs_mcp"
|
||||
|
||||
|
||||
def _truncate_to_max_bytes(text: str, *, max_bytes: int) -> Tuple[str, bool]:
|
||||
blob = text.encode("utf-8")
|
||||
if len(blob) <= max_bytes:
|
||||
return text, False
|
||||
# Reserve a bit for the truncation notice
|
||||
reserve = min(512, max_bytes // 10)
|
||||
head = blob[: max(0, max_bytes - reserve)].decode("utf-8", "replace")
|
||||
head = head.rstrip() + "\n\n[TRUNCATED: response exceeded VM_MCP_MAX_BYTES]\n"
|
||||
return head, True
|
||||
|
||||
|
||||
def _mcp_text_result(text: str, *, is_error: bool = False) -> Dict[str, Any]:
|
||||
text, _truncated = _truncate_to_max_bytes(text, max_bytes=_max_bytes())
|
||||
result: Dict[str, Any] = {"content": [{"type": "text", "text": text}]}
|
||||
if is_error:
|
||||
result["isError"] = True
|
||||
return result
|
||||
|
||||
|
||||
class AkashDocsTools:
|
||||
def __init__(self) -> None:
|
||||
state_dir = Path(os.getenv("VM_AKASH_DOCS_MCP_STATE_DIR") or _default_state_dir())
|
||||
self.store = DocStore(state_dir)
|
||||
|
||||
def akash_docs_list_routes(self) -> Dict[str, Any]:
|
||||
routes = _discover_routes_from_docs_index()
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Discovered {len(routes)} docs route(s) from {AKASH_DOCS_BASE}/.",
|
||||
"data": {"routes": routes},
|
||||
"next_steps": ["akash_docs_fetch(route_or_url=...)"],
|
||||
}
|
||||
|
||||
def akash_docs_fetch(
|
||||
self,
|
||||
*,
|
||||
route_or_url: str,
|
||||
source: str = "auto",
|
||||
ref: str = AKASH_DOCS_GITHUB_REF_DEFAULT,
|
||||
max_chars: int = 12_000,
|
||||
refresh: bool = False,
|
||||
strip_frontmatter: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
route, canonical_url = _normalize_route(route_or_url)
|
||||
source_norm = (source or "auto").strip().lower()
|
||||
if source_norm not in ("auto", "github", "site"):
|
||||
raise ValueError("source must be one of: auto, github, site")
|
||||
|
||||
max_chars_int = max(0, int(max_chars))
|
||||
# Avoid flooding clients; open content_path for full content.
|
||||
max_chars_int = min(max_chars_int, max(2_000, _max_bytes() - 8_000))
|
||||
|
||||
cache_key = f"{source_norm}:{ref}:{route or canonical_url}"
|
||||
cached = self.store.get(cache_key)
|
||||
if cached and not refresh:
|
||||
content = Path(cached.content_path).read_text(encoding="utf-8")
|
||||
if strip_frontmatter and cached.source == "github":
|
||||
content = _strip_frontmatter(content)
|
||||
truncated = len(content) > max_chars_int
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "Returned cached docs content.",
|
||||
"data": {
|
||||
"source": cached.source,
|
||||
"route": cached.route,
|
||||
"url": cached.url,
|
||||
"ref": cached.ref,
|
||||
"cached": True,
|
||||
"fetched_at": cached.fetched_at,
|
||||
"content": content[:max_chars_int],
|
||||
"truncated": truncated,
|
||||
"content_path": cached.content_path,
|
||||
},
|
||||
"next_steps": ["Set refresh=true to refetch."],
|
||||
}
|
||||
|
||||
attempted: List[Dict[str, Any]] = []
|
||||
|
||||
def _try_github() -> Optional[Tuple[str, str, str]]:
|
||||
try:
|
||||
md, raw_url, repo_path = _fetch_markdown_from_github(route, ref=ref)
|
||||
return md, raw_url, repo_path
|
||||
except urllib.error.HTTPError as e:
|
||||
attempted.append({"source": "github", "status": getattr(e, "code", None), "detail": str(e)})
|
||||
return None
|
||||
|
||||
def _try_site() -> Optional[Tuple[str, str]]:
|
||||
try:
|
||||
html = _http_get(canonical_url)
|
||||
article = _extract_article_html(html)
|
||||
text = _html_to_text(article)
|
||||
return text, canonical_url
|
||||
except urllib.error.HTTPError as e:
|
||||
attempted.append({"source": "site", "status": getattr(e, "code", None), "detail": str(e)})
|
||||
return None
|
||||
|
||||
content: str
|
||||
final_source: str
|
||||
final_url: str
|
||||
extra: Dict[str, Any] = {}
|
||||
|
||||
if source_norm in ("auto", "github"):
|
||||
gh = _try_github()
|
||||
if gh:
|
||||
content, final_url, repo_path = gh
|
||||
final_source = "github"
|
||||
extra["repo_path"] = repo_path
|
||||
elif source_norm == "github":
|
||||
raise ValueError("GitHub fetch failed; try source='site' or verify the route/ref.")
|
||||
else:
|
||||
site = _try_site()
|
||||
if not site:
|
||||
raise ValueError(f"Fetch failed for route_or_url={route_or_url!r}. Attempts: {attempted}")
|
||||
content, final_url = site
|
||||
final_source = "site"
|
||||
else:
|
||||
site = _try_site()
|
||||
if not site:
|
||||
raise ValueError(f"Site fetch failed for route_or_url={route_or_url!r}. Attempts: {attempted}")
|
||||
content, final_url = site
|
||||
final_source = "site"
|
||||
|
||||
cached_doc = self.store.save(
|
||||
cache_key=cache_key,
|
||||
source=final_source,
|
||||
route=route,
|
||||
url=final_url,
|
||||
ref=ref,
|
||||
content=content,
|
||||
)
|
||||
|
||||
content_view = content
|
||||
if strip_frontmatter and final_source == "github":
|
||||
content_view = _strip_frontmatter(content_view)
|
||||
truncated = len(content_view) > max_chars_int
|
||||
content_out = content_view[:max_chars_int]
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Fetched docs via {final_source}.",
|
||||
"data": {
|
||||
"source": final_source,
|
||||
"route": route,
|
||||
"url": final_url,
|
||||
"ref": ref,
|
||||
"cached": False,
|
||||
"fetched_at": cached_doc.fetched_at,
|
||||
"content": content_out,
|
||||
"truncated": truncated,
|
||||
"content_path": cached_doc.content_path,
|
||||
"attempts": attempted,
|
||||
**extra,
|
||||
},
|
||||
"next_steps": [
|
||||
"akash_docs_search(query=..., refresh=false)",
|
||||
],
|
||||
}
|
||||
|
||||
def akash_docs_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
refresh: bool = False,
|
||||
ref: str = AKASH_DOCS_GITHUB_REF_DEFAULT,
|
||||
) -> Dict[str, Any]:
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
raise ValueError("query is required")
|
||||
limit = max(1, min(50, int(limit)))
|
||||
|
||||
routes = _discover_routes_from_docs_index()
|
||||
hits: List[Dict[str, Any]] = []
|
||||
|
||||
for route in routes:
|
||||
doc = self.akash_docs_fetch(
|
||||
route_or_url=route,
|
||||
source="github",
|
||||
ref=ref,
|
||||
max_chars=0, # search reads full content from content_path
|
||||
refresh=refresh,
|
||||
strip_frontmatter=True,
|
||||
)
|
||||
data = doc.get("data") or {}
|
||||
content_path = data.get("content_path")
|
||||
if not content_path:
|
||||
continue
|
||||
try:
|
||||
content = Path(str(content_path)).read_text(encoding="utf-8")
|
||||
content = _strip_frontmatter(content)
|
||||
except Exception:
|
||||
continue
|
||||
idx = content.lower().find(q.lower())
|
||||
if idx == -1:
|
||||
continue
|
||||
start = max(0, idx - 80)
|
||||
end = min(len(content), idx + 160)
|
||||
snippet = content[start:end].replace("\n", " ").strip()
|
||||
hits.append(
|
||||
{
|
||||
"route": route,
|
||||
"url": data.get("url"),
|
||||
"source": data.get("source"),
|
||||
"snippet": snippet,
|
||||
}
|
||||
)
|
||||
if len(hits) >= limit:
|
||||
break
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Found {len(hits)} hit(s) across {len(routes)} route(s).",
|
||||
"data": {"query": q, "hits": hits, "routes_searched": len(routes)},
|
||||
"next_steps": ["akash_docs_fetch(route_or_url=hits[0].route)"],
|
||||
}
|
||||
|
||||
def akash_sdl_snippet(
|
||||
self,
|
||||
*,
|
||||
service_name: str,
|
||||
container_image: str,
|
||||
port: int,
|
||||
cpu_units: float = 0.5,
|
||||
memory_size: str = "512Mi",
|
||||
storage_size: str = "512Mi",
|
||||
denom: str = "uakt",
|
||||
price_amount: int = 100,
|
||||
) -> Dict[str, Any]:
|
||||
svc = (service_name or "").strip()
|
||||
img = (container_image or "").strip()
|
||||
if not svc:
|
||||
raise ValueError("service_name is required")
|
||||
if not img:
|
||||
raise ValueError("container_image is required")
|
||||
port_int = int(port)
|
||||
if port_int <= 0 or port_int > 65535:
|
||||
raise ValueError("port must be 1..65535")
|
||||
|
||||
sdl = f"""version: \"2.0\"
|
||||
|
||||
services:
|
||||
{svc}:
|
||||
image: {img}
|
||||
expose:
|
||||
- port: {port_int}
|
||||
to:
|
||||
- global: true
|
||||
|
||||
profiles:
|
||||
compute:
|
||||
{svc}:
|
||||
resources:
|
||||
cpu:
|
||||
units: {cpu_units}
|
||||
memory:
|
||||
size: {memory_size}
|
||||
storage:
|
||||
size: {storage_size}
|
||||
placement:
|
||||
akash:
|
||||
pricing:
|
||||
{svc}:
|
||||
denom: {denom}
|
||||
amount: {int(price_amount)}
|
||||
|
||||
deployment:
|
||||
{svc}:
|
||||
akash:
|
||||
profile: {svc}
|
||||
count: 1
|
||||
"""
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "Generated an Akash SDL template.",
|
||||
"data": {
|
||||
"service_name": svc,
|
||||
"container_image": img,
|
||||
"port": port_int,
|
||||
"sdl": sdl,
|
||||
},
|
||||
"next_steps": [
|
||||
"Save as deploy.yaml and deploy via Akash Console or akash CLI.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
TOOLS: List[Dict[str, Any]] = [
|
||||
{
|
||||
"name": "akash_docs_list_routes",
|
||||
"description": "Discover common Akash docs routes by scraping https://akash.network/docs/ (SSR HTML).",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
{
|
||||
"name": "akash_docs_fetch",
|
||||
"description": "Fetch an Akash docs page (prefers GitHub markdown in akash-network/website-revamp; falls back to site HTML).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"route_or_url": {"type": "string"},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "auto|github|site",
|
||||
"default": "auto",
|
||||
},
|
||||
"ref": {"type": "string", "default": AKASH_DOCS_GITHUB_REF_DEFAULT},
|
||||
"max_chars": {"type": "integer", "default": 12000},
|
||||
"refresh": {"type": "boolean", "default": False},
|
||||
"strip_frontmatter": {"type": "boolean", "default": True},
|
||||
},
|
||||
"required": ["route_or_url"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "akash_docs_search",
|
||||
"description": "Keyword search across routes discovered from /docs (fetches + caches GitHub markdown).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"limit": {"type": "integer", "default": 10},
|
||||
"refresh": {"type": "boolean", "default": False},
|
||||
"ref": {"type": "string", "default": AKASH_DOCS_GITHUB_REF_DEFAULT},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "akash_sdl_snippet",
|
||||
"description": "Generate a minimal Akash SDL manifest for a single service exposing one port.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"service_name": {"type": "string"},
|
||||
"container_image": {"type": "string"},
|
||||
"port": {"type": "integer"},
|
||||
"cpu_units": {"type": "number", "default": 0.5},
|
||||
"memory_size": {"type": "string", "default": "512Mi"},
|
||||
"storage_size": {"type": "string", "default": "512Mi"},
|
||||
"denom": {"type": "string", "default": "uakt"},
|
||||
"price_amount": {"type": "integer", "default": 100},
|
||||
},
|
||||
"required": ["service_name", "container_image", "port"],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class StdioJsonRpc:
|
||||
def __init__(self) -> None:
|
||||
self._in = sys.stdin.buffer
|
||||
self._out = sys.stdout.buffer
|
||||
self._mode: str | None = None # "headers" | "line"
|
||||
|
||||
def read_message(self) -> Optional[Dict[str, Any]]:
|
||||
while True:
|
||||
if self._mode == "line":
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
raw = line.decode("utf-8", "replace").strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
continue
|
||||
|
||||
first = self._in.readline()
|
||||
if not first:
|
||||
return None
|
||||
|
||||
if first in (b"\r\n", b"\n"):
|
||||
continue
|
||||
|
||||
# Auto-detect newline-delimited JSON framing.
|
||||
if self._mode is None and first.lstrip().startswith(b"{"):
|
||||
try:
|
||||
msg = json.loads(first.decode("utf-8", "replace"))
|
||||
except Exception:
|
||||
msg = None
|
||||
if isinstance(msg, dict):
|
||||
self._mode = "line"
|
||||
return msg
|
||||
|
||||
headers: Dict[str, str] = {}
|
||||
try:
|
||||
text = first.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
while True:
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
if line in (b"\r\n", b"\n"):
|
||||
break
|
||||
try:
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
if "content-length" not in headers:
|
||||
return None
|
||||
try:
|
||||
length = int(headers["content-length"])
|
||||
except ValueError:
|
||||
return None
|
||||
body = self._in.read(length)
|
||||
if not body:
|
||||
return None
|
||||
self._mode = "headers"
|
||||
msg = json.loads(body.decode("utf-8", "replace"))
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
return None
|
||||
|
||||
def write_message(self, message: Dict[str, Any]) -> None:
|
||||
if self._mode == "line":
|
||||
payload = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
self._out.write(payload + b"\n")
|
||||
self._out.flush()
|
||||
return
|
||||
|
||||
body = json.dumps(message, ensure_ascii=False, separators=(",", ":")).encode(
|
||||
"utf-8"
|
||||
)
|
||||
header = f"Content-Length: {len(body)}\r\n\r\n".encode("utf-8")
|
||||
self._out.write(header)
|
||||
self._out.write(body)
|
||||
self._out.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tools = AkashDocsTools()
|
||||
rpc = StdioJsonRpc()
|
||||
|
||||
handlers: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {
|
||||
"akash_docs_list_routes": lambda a: tools.akash_docs_list_routes(),
|
||||
"akash_docs_fetch": lambda a: tools.akash_docs_fetch(**a),
|
||||
"akash_docs_search": lambda a: tools.akash_docs_search(**a),
|
||||
"akash_sdl_snippet": lambda a: tools.akash_sdl_snippet(**a),
|
||||
}
|
||||
|
||||
while True:
|
||||
msg = rpc.read_message()
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
method = msg.get("method")
|
||||
msg_id = msg.get("id")
|
||||
params = msg.get("params") or {}
|
||||
|
||||
try:
|
||||
if method == "initialize":
|
||||
result = {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {"name": "akash_docs", "version": "0.1.0"},
|
||||
"capabilities": {"tools": {}},
|
||||
}
|
||||
rpc.write_message({"jsonrpc": "2.0", "id": msg_id, "result": result})
|
||||
continue
|
||||
|
||||
if method == "tools/list":
|
||||
rpc.write_message(
|
||||
{"jsonrpc": "2.0", "id": msg_id, "result": {"tools": TOOLS}}
|
||||
)
|
||||
continue
|
||||
|
||||
if method == "tools/call":
|
||||
tool_name = str(params.get("name") or "")
|
||||
args = params.get("arguments") or {}
|
||||
if tool_name not in handlers:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
f"Unknown tool: {tool_name}\nKnown tools: {', '.join(sorted(handlers.keys()))}",
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = handlers[tool_name](args)
|
||||
# Split payload: meta JSON + optional raw content.
|
||||
# If payload["data"]["content"] exists, emit it as a second text block for readability.
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
content_text = None
|
||||
if isinstance(data, dict) and isinstance(data.get("content"), str):
|
||||
content_text = data["content"]
|
||||
data = dict(data)
|
||||
data.pop("content", None)
|
||||
payload = dict(payload)
|
||||
payload["data"] = data
|
||||
|
||||
blocks = [json.dumps(payload, ensure_ascii=False, indent=2)]
|
||||
if content_text:
|
||||
blocks.append(content_text)
|
||||
result: Dict[str, Any] = {
|
||||
"content": [{"type": "text", "text": b} for b in blocks]
|
||||
}
|
||||
rpc.write_message({"jsonrpc": "2.0", "id": msg_id, "result": result})
|
||||
except Exception as e: # noqa: BLE001
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
f"Error: {e}",
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Ignore notifications.
|
||||
if msg_id is None:
|
||||
continue
|
||||
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
f"Unsupported method: {method}",
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Last-resort: avoid crashing the server.
|
||||
if msg_id is not None:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(f"fatal error: {e}", is_error=True),
|
||||
}
|
||||
)
|
||||
11
mcp/cloudflare_safe/__init__.py
Normal file
11
mcp/cloudflare_safe/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
cloudflare_safe MCP server.
|
||||
|
||||
Summary-first Cloudflare tooling with hard output caps and default redaction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
6
mcp/cloudflare_safe/__main__.py
Normal file
6
mcp/cloudflare_safe/__main__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
496
mcp/cloudflare_safe/cloudflare_api.py
Normal file
496
mcp/cloudflare_safe/cloudflare_api.py
Normal file
@@ -0,0 +1,496 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
MutableMapping,
|
||||
Optional,
|
||||
Sequence,
|
||||
Tuple,
|
||||
)
|
||||
|
||||
CF_API_BASE = "https://api.cloudflare.com/client/v4"
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def stable_hash(data: Any) -> str:
|
||||
blob = json.dumps(
|
||||
data, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(blob).hexdigest()
|
||||
|
||||
|
||||
class CloudflareError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CloudflareContext:
|
||||
api_token: str
|
||||
account_id: str
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> "CloudflareContext":
|
||||
api_token = (
|
||||
os.getenv("CLOUDFLARE_API_TOKEN")
|
||||
or os.getenv("CF_API_TOKEN")
|
||||
or os.getenv("CLOUDFLARE_TOKEN")
|
||||
or ""
|
||||
).strip()
|
||||
account_id = (
|
||||
os.getenv("CLOUDFLARE_ACCOUNT_ID") or os.getenv("CF_ACCOUNT_ID") or ""
|
||||
).strip()
|
||||
|
||||
if not api_token:
|
||||
raise CloudflareError(
|
||||
"Missing Cloudflare API token. Set CLOUDFLARE_API_TOKEN (or CF_API_TOKEN)."
|
||||
)
|
||||
if not account_id:
|
||||
raise CloudflareError(
|
||||
"Missing Cloudflare account id. Set CLOUDFLARE_ACCOUNT_ID (or CF_ACCOUNT_ID)."
|
||||
)
|
||||
return CloudflareContext(api_token=api_token, account_id=account_id)
|
||||
|
||||
|
||||
class CloudflareClient:
|
||||
def __init__(self, *, api_token: str) -> None:
|
||||
self.api_token = api_token
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
params: Optional[Mapping[str, str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
url = f"{CF_API_BASE}{path}"
|
||||
if params:
|
||||
url = f"{url}?{urllib.parse.urlencode(params)}"
|
||||
|
||||
req = urllib.request.Request(
|
||||
url=url,
|
||||
method=method,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_token}",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read() if hasattr(e, "read") else b""
|
||||
detail = raw.decode("utf-8", "replace")
|
||||
raise CloudflareError(
|
||||
f"Cloudflare API HTTP {e.code} for {path}: {detail}"
|
||||
) from e
|
||||
except urllib.error.URLError as e:
|
||||
raise CloudflareError(
|
||||
f"Cloudflare API request failed for {path}: {e}"
|
||||
) from e
|
||||
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8", "replace"))
|
||||
except json.JSONDecodeError:
|
||||
raise CloudflareError(
|
||||
f"Cloudflare API returned non-JSON for {path}: {raw[:200]!r}"
|
||||
)
|
||||
|
||||
if not data.get("success", True):
|
||||
raise CloudflareError(
|
||||
f"Cloudflare API error for {path}: {data.get('errors')}"
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
params: Optional[Mapping[str, str]] = None,
|
||||
per_page: int = 100,
|
||||
max_pages: int = 5,
|
||||
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""
|
||||
Fetch a paginated Cloudflare endpoint.
|
||||
|
||||
Returns (results, result_info).
|
||||
"""
|
||||
results: List[Dict[str, Any]] = []
|
||||
page = 1
|
||||
last_info: Dict[str, Any] = {}
|
||||
|
||||
while True:
|
||||
merged_params: Dict[str, str] = {
|
||||
"page": str(page),
|
||||
"per_page": str(per_page),
|
||||
}
|
||||
if params:
|
||||
merged_params.update({k: str(v) for k, v in params.items()})
|
||||
|
||||
data = self._request("GET", path, params=merged_params)
|
||||
batch = data.get("result") or []
|
||||
if not isinstance(batch, list):
|
||||
batch = [batch]
|
||||
results.extend(batch)
|
||||
last_info = data.get("result_info") or {}
|
||||
|
||||
total_pages = int(last_info.get("total_pages") or 1)
|
||||
if page >= total_pages or page >= max_pages:
|
||||
break
|
||||
page += 1
|
||||
|
||||
return results, last_info
|
||||
|
||||
def list_zones(self) -> List[Dict[str, Any]]:
|
||||
zones, _info = self.paginate("/zones", max_pages=2)
|
||||
return zones
|
||||
|
||||
def list_dns_records_summary(
|
||||
self, zone_id: str, *, max_pages: int = 1
|
||||
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
||||
return self.paginate(f"/zones/{zone_id}/dns_records", max_pages=max_pages)
|
||||
|
||||
def list_tunnels(self, account_id: str) -> List[Dict[str, Any]]:
|
||||
tunnels, _info = self.paginate(
|
||||
f"/accounts/{account_id}/cfd_tunnel", max_pages=2
|
||||
)
|
||||
return tunnels
|
||||
|
||||
def list_tunnel_connections(
|
||||
self, account_id: str, tunnel_id: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
data = self._request(
|
||||
"GET", f"/accounts/{account_id}/cfd_tunnel/{tunnel_id}/connections"
|
||||
)
|
||||
result = data.get("result") or []
|
||||
return result if isinstance(result, list) else [result]
|
||||
|
||||
def list_access_apps(self, account_id: str) -> List[Dict[str, Any]]:
|
||||
apps, _info = self.paginate(f"/accounts/{account_id}/access/apps", max_pages=3)
|
||||
return apps
|
||||
|
||||
def list_access_policies(
|
||||
self, account_id: str, app_id: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
policies, _info = self.paginate(
|
||||
f"/accounts/{account_id}/access/apps/{app_id}/policies",
|
||||
max_pages=3,
|
||||
)
|
||||
return policies
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SnapshotMeta:
|
||||
snapshot_id: str
|
||||
created_at: str
|
||||
scopes: List[str]
|
||||
snapshot_path: str
|
||||
|
||||
|
||||
class SnapshotStore:
|
||||
def __init__(self, root_dir: Path) -> None:
|
||||
self.root_dir = root_dir
|
||||
self.snapshots_dir = root_dir / "snapshots"
|
||||
self.diffs_dir = root_dir / "diffs"
|
||||
self.snapshots_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.diffs_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._index: Dict[str, SnapshotMeta] = {}
|
||||
|
||||
def get(self, snapshot_id: str) -> SnapshotMeta:
|
||||
if snapshot_id not in self._index:
|
||||
raise CloudflareError(f"Unknown snapshot_id: {snapshot_id}")
|
||||
return self._index[snapshot_id]
|
||||
|
||||
def load_snapshot(self, snapshot_id: str) -> Dict[str, Any]:
|
||||
meta = self.get(snapshot_id)
|
||||
return json.loads(Path(meta.snapshot_path).read_text(encoding="utf-8"))
|
||||
|
||||
def create_snapshot(
|
||||
self,
|
||||
*,
|
||||
client: CloudflareClient,
|
||||
ctx: CloudflareContext,
|
||||
scopes: Sequence[str],
|
||||
zone_id: Optional[str] = None,
|
||||
zone_name: Optional[str] = None,
|
||||
dns_max_pages: int = 1,
|
||||
) -> Tuple[SnapshotMeta, Dict[str, Any]]:
|
||||
scopes_norm = sorted(set(scopes))
|
||||
created_at = utc_now_iso()
|
||||
|
||||
zones = client.list_zones()
|
||||
zones_min = [
|
||||
{
|
||||
"id": z.get("id"),
|
||||
"name": z.get("name"),
|
||||
"status": z.get("status"),
|
||||
"paused": z.get("paused"),
|
||||
}
|
||||
for z in zones
|
||||
]
|
||||
|
||||
selected_zone_id = zone_id
|
||||
if not selected_zone_id and zone_name:
|
||||
for z in zones_min:
|
||||
if z.get("name") == zone_name:
|
||||
selected_zone_id = str(z.get("id"))
|
||||
break
|
||||
|
||||
snapshot: Dict[str, Any] = {
|
||||
"meta": {
|
||||
"snapshot_id": "",
|
||||
"created_at": created_at,
|
||||
"account_id": ctx.account_id,
|
||||
"scopes": scopes_norm,
|
||||
},
|
||||
"zones": zones_min,
|
||||
}
|
||||
|
||||
if "tunnels" in scopes_norm:
|
||||
tunnels = client.list_tunnels(ctx.account_id)
|
||||
tunnels_min: List[Dict[str, Any]] = []
|
||||
for t in tunnels:
|
||||
tid = t.get("id")
|
||||
name = t.get("name")
|
||||
status = t.get("status")
|
||||
connector_count: Optional[int] = None
|
||||
last_seen: Optional[str] = None
|
||||
if tid and status != "deleted":
|
||||
conns = client.list_tunnel_connections(ctx.account_id, str(tid))
|
||||
connector_count = len(conns)
|
||||
# Pick the most recent 'opened_at' if present.
|
||||
opened = [c.get("opened_at") for c in conns if isinstance(c, dict)]
|
||||
opened = [o for o in opened if isinstance(o, str)]
|
||||
last_seen = max(opened) if opened else None
|
||||
|
||||
tunnels_min.append(
|
||||
{
|
||||
"id": tid,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"created_at": t.get("created_at"),
|
||||
"deleted_at": t.get("deleted_at"),
|
||||
"connector_count": connector_count,
|
||||
"last_seen": last_seen,
|
||||
}
|
||||
)
|
||||
snapshot["tunnels"] = tunnels_min
|
||||
|
||||
if "access_apps" in scopes_norm:
|
||||
apps = client.list_access_apps(ctx.account_id)
|
||||
apps_min = [
|
||||
{
|
||||
"id": a.get("id"),
|
||||
"name": a.get("name"),
|
||||
"domain": a.get("domain"),
|
||||
"type": a.get("type"),
|
||||
"created_at": a.get("created_at"),
|
||||
"updated_at": a.get("updated_at"),
|
||||
}
|
||||
for a in apps
|
||||
]
|
||||
snapshot["access_apps"] = apps_min
|
||||
|
||||
if "dns" in scopes_norm:
|
||||
if selected_zone_id:
|
||||
records, info = client.list_dns_records_summary(
|
||||
selected_zone_id, max_pages=dns_max_pages
|
||||
)
|
||||
records_min = [
|
||||
{
|
||||
"id": r.get("id"),
|
||||
"type": r.get("type"),
|
||||
"name": r.get("name"),
|
||||
"content": r.get("content"),
|
||||
"proxied": r.get("proxied"),
|
||||
"ttl": r.get("ttl"),
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
snapshot["dns"] = {
|
||||
"zone_id": selected_zone_id,
|
||||
"zone_name": zone_name,
|
||||
"result_info": info,
|
||||
"records_sample": records_min,
|
||||
}
|
||||
else:
|
||||
snapshot["dns"] = {
|
||||
"note": "dns scope requested but no zone_id/zone_name provided; only zones list included",
|
||||
}
|
||||
|
||||
snapshot_id = f"cf_{created_at.replace(':', '').replace('-', '').replace('.', '')}_{stable_hash(snapshot)[:10]}"
|
||||
snapshot["meta"]["snapshot_id"] = snapshot_id
|
||||
|
||||
path = self.snapshots_dir / f"{snapshot_id}.json"
|
||||
path.write_text(
|
||||
json.dumps(snapshot, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
meta = SnapshotMeta(
|
||||
snapshot_id=snapshot_id,
|
||||
created_at=created_at,
|
||||
scopes=scopes_norm,
|
||||
snapshot_path=str(path),
|
||||
)
|
||||
self._index[snapshot_id] = meta
|
||||
return meta, snapshot
|
||||
|
||||
def diff(
|
||||
self,
|
||||
*,
|
||||
from_snapshot_id: str,
|
||||
to_snapshot_id: str,
|
||||
scopes: Optional[Sequence[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
before = self.load_snapshot(from_snapshot_id)
|
||||
after = self.load_snapshot(to_snapshot_id)
|
||||
|
||||
scopes_before = set(before.get("meta", {}).get("scopes") or [])
|
||||
scopes_after = set(after.get("meta", {}).get("scopes") or [])
|
||||
scopes_all = sorted(scopes_before | scopes_after)
|
||||
scopes_use = sorted(set(scopes or scopes_all))
|
||||
|
||||
def index_by_id(
|
||||
items: Iterable[Mapping[str, Any]],
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
out: Dict[str, Dict[str, Any]] = {}
|
||||
for it in items:
|
||||
_id = it.get("id")
|
||||
if _id is None:
|
||||
continue
|
||||
out[str(_id)] = dict(it)
|
||||
return out
|
||||
|
||||
diff_out: Dict[str, Any] = {
|
||||
"from": from_snapshot_id,
|
||||
"to": to_snapshot_id,
|
||||
"scopes": scopes_use,
|
||||
"changes": {},
|
||||
}
|
||||
|
||||
for scope in scopes_use:
|
||||
if scope not in {"tunnels", "access_apps", "zones"}:
|
||||
continue
|
||||
b_items = before.get(scope) or []
|
||||
a_items = after.get(scope) or []
|
||||
if not isinstance(b_items, list) or not isinstance(a_items, list):
|
||||
continue
|
||||
b_map = index_by_id(b_items)
|
||||
a_map = index_by_id(a_items)
|
||||
added = [a_map[k] for k in sorted(set(a_map) - set(b_map))]
|
||||
removed = [b_map[k] for k in sorted(set(b_map) - set(a_map))]
|
||||
|
||||
changed: List[Dict[str, Any]] = []
|
||||
for k in sorted(set(a_map) & set(b_map)):
|
||||
if stable_hash(a_map[k]) != stable_hash(b_map[k]):
|
||||
changed.append({"id": k, "before": b_map[k], "after": a_map[k]})
|
||||
|
||||
diff_out["changes"][scope] = {
|
||||
"added": [{"id": x.get("id"), "name": x.get("name")} for x in added],
|
||||
"removed": [
|
||||
{"id": x.get("id"), "name": x.get("name")} for x in removed
|
||||
],
|
||||
"changed": [
|
||||
{"id": x.get("id"), "name": x.get("after", {}).get("name")}
|
||||
for x in changed
|
||||
],
|
||||
"counts": {
|
||||
"added": len(added),
|
||||
"removed": len(removed),
|
||||
"changed": len(changed),
|
||||
},
|
||||
}
|
||||
|
||||
diff_path = self.diffs_dir / f"{from_snapshot_id}_to_{to_snapshot_id}.json"
|
||||
diff_path.write_text(
|
||||
json.dumps(diff_out, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
diff_out["diff_path"] = str(diff_path)
|
||||
return diff_out
|
||||
|
||||
|
||||
def parse_cloudflared_config_ingress(config_text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Best-effort parser for cloudflared YAML config ingress rules.
|
||||
|
||||
We intentionally avoid a YAML dependency; this extracts common patterns:
|
||||
- hostname: example.com
|
||||
service: http://127.0.0.1:8080
|
||||
"""
|
||||
rules: List[Dict[str, str]] = []
|
||||
lines = config_text.splitlines()
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
stripped = line.lstrip()
|
||||
if not stripped.startswith("-"):
|
||||
i += 1
|
||||
continue
|
||||
after_dash = stripped[1:].lstrip()
|
||||
if not after_dash.startswith("hostname:"):
|
||||
i += 1
|
||||
continue
|
||||
|
||||
hostname = after_dash[len("hostname:") :].strip().strip('"').strip("'")
|
||||
base_indent = len(line) - len(line.lstrip())
|
||||
|
||||
service = ""
|
||||
j = i + 1
|
||||
while j < len(lines):
|
||||
next_line = lines[j]
|
||||
if next_line.strip() == "":
|
||||
j += 1
|
||||
continue
|
||||
|
||||
next_indent = len(next_line) - len(next_line.lstrip())
|
||||
if next_indent <= base_indent:
|
||||
break
|
||||
|
||||
next_stripped = next_line.lstrip()
|
||||
if next_stripped.startswith("service:"):
|
||||
service = next_stripped[len("service:") :].strip().strip('"').strip("'")
|
||||
break
|
||||
j += 1
|
||||
|
||||
rules.append({"hostname": hostname, "service": service})
|
||||
i = j
|
||||
return rules
|
||||
|
||||
|
||||
def ingress_summary_from_file(
|
||||
*,
|
||||
config_path: str,
|
||||
max_rules: int = 50,
|
||||
) -> Dict[str, Any]:
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
raise CloudflareError(f"cloudflared config not found: {config_path}")
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
rules = parse_cloudflared_config_ingress(text)
|
||||
hostnames = sorted({r["hostname"] for r in rules if r.get("hostname")})
|
||||
return {
|
||||
"config_path": config_path,
|
||||
"ingress_rule_count": len(rules),
|
||||
"hostnames": hostnames[:max_rules],
|
||||
"rules_sample": rules[:max_rules],
|
||||
"truncated": len(rules) > max_rules,
|
||||
}
|
||||
725
mcp/cloudflare_safe/server.py
Normal file
725
mcp/cloudflare_safe/server.py
Normal file
@@ -0,0 +1,725 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
from .cloudflare_api import (
|
||||
CloudflareClient,
|
||||
CloudflareContext,
|
||||
CloudflareError,
|
||||
SnapshotStore,
|
||||
ingress_summary_from_file,
|
||||
)
|
||||
|
||||
MAX_BYTES_DEFAULT = 32_000
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
# server.py -> cloudflare_safe -> mcp -> <repo root>
|
||||
return Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _max_bytes() -> int:
|
||||
raw = (os.getenv("VM_MCP_MAX_BYTES") or "").strip()
|
||||
if not raw:
|
||||
return MAX_BYTES_DEFAULT
|
||||
try:
|
||||
return max(4_096, int(raw))
|
||||
except ValueError:
|
||||
return MAX_BYTES_DEFAULT
|
||||
|
||||
|
||||
def _redact(obj: Any) -> Any:
|
||||
sensitive_keys = ("token", "secret", "password", "private", "key", "certificate")
|
||||
|
||||
if isinstance(obj, dict):
|
||||
out: Dict[str, Any] = {}
|
||||
for k, v in obj.items():
|
||||
if any(s in str(k).lower() for s in sensitive_keys):
|
||||
out[k] = "<REDACTED>"
|
||||
else:
|
||||
out[k] = _redact(v)
|
||||
return out
|
||||
if isinstance(obj, list):
|
||||
return [_redact(v) for v in obj]
|
||||
if isinstance(obj, str):
|
||||
if obj.startswith("ghp_") or obj.startswith("github_pat_"):
|
||||
return "<REDACTED>"
|
||||
return obj
|
||||
return obj
|
||||
|
||||
|
||||
def _safe_json(payload: Dict[str, Any]) -> str:
|
||||
payload = _redact(payload)
|
||||
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
if len(raw.encode("utf-8")) <= _max_bytes():
|
||||
return json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
|
||||
# Truncate: keep only summary + next_steps.
|
||||
truncated = {
|
||||
"ok": payload.get("ok", True),
|
||||
"truncated": True,
|
||||
"summary": payload.get("summary", "Response exceeded max size; truncated."),
|
||||
"next_steps": payload.get(
|
||||
"next_steps",
|
||||
[
|
||||
"request a narrower scope (e.g., scopes=['tunnels'])",
|
||||
"request an export path instead of inline content",
|
||||
],
|
||||
),
|
||||
}
|
||||
return json.dumps(truncated, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _mcp_text_result(
|
||||
payload: Dict[str, Any], *, is_error: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": [{"type": "text", "text": _safe_json(payload)}]
|
||||
}
|
||||
if is_error:
|
||||
result["isError"] = True
|
||||
return result
|
||||
|
||||
|
||||
def _default_state_dir() -> Path:
|
||||
return _repo_root() / "archive_runtime" / "cloudflare_mcp"
|
||||
|
||||
|
||||
class CloudflareSafeTools:
|
||||
def __init__(self) -> None:
|
||||
self.store = SnapshotStore(
|
||||
Path(os.getenv("VM_CF_MCP_STATE_DIR") or _default_state_dir())
|
||||
)
|
||||
|
||||
def cf_snapshot(
|
||||
self,
|
||||
*,
|
||||
scopes: Optional[Sequence[str]] = None,
|
||||
zone_id: Optional[str] = None,
|
||||
zone_name: Optional[str] = None,
|
||||
dns_max_pages: int = 1,
|
||||
) -> Dict[str, Any]:
|
||||
scopes_use = list(scopes or ["tunnels", "access_apps"])
|
||||
ctx = CloudflareContext.from_env()
|
||||
client = CloudflareClient(api_token=ctx.api_token)
|
||||
meta, snapshot = self.store.create_snapshot(
|
||||
client=client,
|
||||
ctx=ctx,
|
||||
scopes=scopes_use,
|
||||
zone_id=zone_id,
|
||||
zone_name=zone_name,
|
||||
dns_max_pages=dns_max_pages,
|
||||
)
|
||||
|
||||
summary = (
|
||||
f"Snapshot {meta.snapshot_id} captured "
|
||||
f"(scopes={','.join(meta.scopes)}) and written to {meta.snapshot_path}."
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": summary,
|
||||
"data": {
|
||||
"snapshot_id": meta.snapshot_id,
|
||||
"created_at": meta.created_at,
|
||||
"scopes": meta.scopes,
|
||||
"snapshot_path": meta.snapshot_path,
|
||||
"counts": {
|
||||
"zones": len(snapshot.get("zones") or []),
|
||||
"tunnels": len(snapshot.get("tunnels") or []),
|
||||
"access_apps": len(snapshot.get("access_apps") or []),
|
||||
},
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"cf_config_diff(from_snapshot_id=..., to_snapshot_id=...)",
|
||||
"cf_export_config(full=false, snapshot_id=...)",
|
||||
],
|
||||
}
|
||||
|
||||
def cf_refresh(
|
||||
self,
|
||||
*,
|
||||
snapshot_id: str,
|
||||
scopes: Optional[Sequence[str]] = None,
|
||||
dns_max_pages: int = 1,
|
||||
) -> Dict[str, Any]:
|
||||
before_meta = self.store.get(snapshot_id)
|
||||
before = self.store.load_snapshot(snapshot_id)
|
||||
scopes_use = list(scopes or (before.get("meta", {}).get("scopes") or []))
|
||||
|
||||
ctx = CloudflareContext.from_env()
|
||||
client = CloudflareClient(api_token=ctx.api_token)
|
||||
|
||||
meta, _snapshot = self.store.create_snapshot(
|
||||
client=client,
|
||||
ctx=ctx,
|
||||
scopes=scopes_use,
|
||||
zone_id=(before.get("dns") or {}).get("zone_id"),
|
||||
zone_name=(before.get("dns") or {}).get("zone_name"),
|
||||
dns_max_pages=dns_max_pages,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Refreshed {before_meta.snapshot_id} -> {meta.snapshot_id} (scopes={','.join(meta.scopes)}).",
|
||||
"data": {
|
||||
"from_snapshot_id": before_meta.snapshot_id,
|
||||
"to_snapshot_id": meta.snapshot_id,
|
||||
"snapshot_path": meta.snapshot_path,
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"cf_config_diff(from_snapshot_id=..., to_snapshot_id=...)",
|
||||
],
|
||||
}
|
||||
|
||||
def cf_config_diff(
|
||||
self,
|
||||
*,
|
||||
from_snapshot_id: str,
|
||||
to_snapshot_id: str,
|
||||
scopes: Optional[Sequence[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
diff = self.store.diff(
|
||||
from_snapshot_id=from_snapshot_id,
|
||||
to_snapshot_id=to_snapshot_id,
|
||||
scopes=scopes,
|
||||
)
|
||||
|
||||
# Keep the response small; point to diff_path for full detail.
|
||||
changes = diff.get("changes") or {}
|
||||
counts = {
|
||||
scope: (changes.get(scope) or {}).get("counts")
|
||||
for scope in sorted(changes.keys())
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Diff computed and written to {diff.get('diff_path')}.",
|
||||
"data": {
|
||||
"from_snapshot_id": from_snapshot_id,
|
||||
"to_snapshot_id": to_snapshot_id,
|
||||
"scopes": diff.get("scopes"),
|
||||
"counts": counts,
|
||||
"diff_path": diff.get("diff_path"),
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Use filesystem MCP to open diff_path for full details",
|
||||
"Run cf_export_config(full=false, snapshot_id=...) for a safe export path",
|
||||
],
|
||||
}
|
||||
|
||||
def cf_export_config(
|
||||
self,
|
||||
*,
|
||||
snapshot_id: Optional[str] = None,
|
||||
full: bool = False,
|
||||
scopes: Optional[Sequence[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if snapshot_id is None:
|
||||
snap = self.cf_snapshot(scopes=scopes)
|
||||
snapshot_id = str((snap.get("data") or {}).get("snapshot_id"))
|
||||
|
||||
meta = self.store.get(snapshot_id)
|
||||
if not full:
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "Export is summary-first; full config requires full=true.",
|
||||
"data": {
|
||||
"snapshot_id": meta.snapshot_id,
|
||||
"snapshot_path": meta.snapshot_path,
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Use filesystem MCP to open snapshot_path",
|
||||
"If you truly need inline data, call cf_export_config(full=true, snapshot_id=...)",
|
||||
],
|
||||
}
|
||||
|
||||
snapshot = self.store.load_snapshot(snapshot_id)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "Full snapshot export (redacted + size-capped). Prefer snapshot_path for large data.",
|
||||
"data": snapshot,
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
f"Snapshot file: {meta.snapshot_path}",
|
||||
],
|
||||
}
|
||||
|
||||
def cf_tunnel_status(
|
||||
self,
|
||||
*,
|
||||
snapshot_id: Optional[str] = None,
|
||||
tunnel_name: Optional[str] = None,
|
||||
tunnel_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if snapshot_id:
|
||||
snap = self.store.load_snapshot(snapshot_id)
|
||||
tunnels = snap.get("tunnels") or []
|
||||
else:
|
||||
snap = self.cf_snapshot(scopes=["tunnels"])
|
||||
sid = str((snap.get("data") or {}).get("snapshot_id"))
|
||||
tunnels = self.store.load_snapshot(sid).get("tunnels") or []
|
||||
|
||||
def matches(t: Dict[str, Any]) -> bool:
|
||||
if tunnel_id and str(t.get("id")) != str(tunnel_id):
|
||||
return False
|
||||
if tunnel_name and str(t.get("name")) != str(tunnel_name):
|
||||
return False
|
||||
return True
|
||||
|
||||
filtered = [t for t in tunnels if isinstance(t, dict) and matches(t)]
|
||||
if not filtered and (tunnel_id or tunnel_name):
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": "Tunnel not found in snapshot.",
|
||||
"data": {"tunnel_id": tunnel_id, "tunnel_name": tunnel_name},
|
||||
"truncated": False,
|
||||
"next_steps": ["Call cf_snapshot(scopes=['tunnels']) and retry."],
|
||||
}
|
||||
|
||||
connectors = [t.get("connector_count") for t in filtered if isinstance(t, dict)]
|
||||
connectors = [c for c in connectors if isinstance(c, int)]
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Returned {len(filtered)} tunnel(s).",
|
||||
"data": {
|
||||
"tunnels": [
|
||||
{
|
||||
"id": t.get("id"),
|
||||
"name": t.get("name"),
|
||||
"status": t.get("status"),
|
||||
"connector_count": t.get("connector_count"),
|
||||
"last_seen": t.get("last_seen"),
|
||||
}
|
||||
for t in filtered
|
||||
],
|
||||
"connectors_total": sum(connectors) if connectors else 0,
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"For local ingress hostnames, use cf_tunnel_ingress_summary(config_path='/etc/cloudflared/config.yml')",
|
||||
],
|
||||
}
|
||||
|
||||
def cf_tunnel_ingress_summary(
|
||||
self,
|
||||
*,
|
||||
config_path: str = "/etc/cloudflared/config.yml",
|
||||
full: bool = False,
|
||||
max_rules: int = 50,
|
||||
) -> Dict[str, Any]:
|
||||
summary = ingress_summary_from_file(
|
||||
config_path=config_path, max_rules=max_rules
|
||||
)
|
||||
if not full:
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Parsed ingress hostnames from {config_path}.",
|
||||
"data": {
|
||||
"config_path": summary["config_path"],
|
||||
"ingress_rule_count": summary["ingress_rule_count"],
|
||||
"hostnames": summary["hostnames"],
|
||||
"truncated": summary["truncated"],
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Call cf_tunnel_ingress_summary(full=true, ...) to include service mappings (still capped).",
|
||||
],
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Ingress summary (full=true) for {config_path}.",
|
||||
"data": summary,
|
||||
"truncated": False,
|
||||
"next_steps": [],
|
||||
}
|
||||
|
||||
def cf_access_policy_list(
|
||||
self,
|
||||
*,
|
||||
app_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
ctx = CloudflareContext.from_env()
|
||||
client = CloudflareClient(api_token=ctx.api_token)
|
||||
|
||||
if not app_id:
|
||||
apps = client.list_access_apps(ctx.account_id)
|
||||
apps_min = [
|
||||
{
|
||||
"id": a.get("id"),
|
||||
"name": a.get("name"),
|
||||
"domain": a.get("domain"),
|
||||
"type": a.get("type"),
|
||||
}
|
||||
for a in apps
|
||||
]
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Returned {len(apps_min)} Access app(s). Provide app_id to list policies.",
|
||||
"data": {"apps": apps_min},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Call cf_access_policy_list(app_id=...)",
|
||||
],
|
||||
}
|
||||
|
||||
policies = client.list_access_policies(ctx.account_id, app_id)
|
||||
policies_min = [
|
||||
{
|
||||
"id": p.get("id"),
|
||||
"name": p.get("name"),
|
||||
"decision": p.get("decision"),
|
||||
"precedence": p.get("precedence"),
|
||||
}
|
||||
for p in policies
|
||||
]
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Returned {len(policies_min)} policy/policies for app_id={app_id}.",
|
||||
"data": {"app_id": app_id, "policies": policies_min},
|
||||
"truncated": False,
|
||||
"next_steps": [],
|
||||
}
|
||||
|
||||
|
||||
TOOLS: List[Dict[str, Any]] = [
|
||||
{
|
||||
"name": "cf_snapshot",
|
||||
"description": "Create a summary-first Cloudflare state snapshot (writes JSON to disk; returns snapshot_id + paths).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"scopes": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Scopes to fetch (default: ['tunnels','access_apps']). Supported: zones,tunnels,access_apps,dns",
|
||||
},
|
||||
"zone_id": {"type": "string"},
|
||||
"zone_name": {"type": "string"},
|
||||
"dns_max_pages": {"type": "integer", "default": 1},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_refresh",
|
||||
"description": "Refresh a prior snapshot (creates a new snapshot_id).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"snapshot_id": {"type": "string"},
|
||||
"scopes": {"type": "array", "items": {"type": "string"}},
|
||||
"dns_max_pages": {"type": "integer", "default": 1},
|
||||
},
|
||||
"required": ["snapshot_id"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_config_diff",
|
||||
"description": "Diff two snapshots (summary counts inline; full diff written to disk).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_snapshot_id": {"type": "string"},
|
||||
"to_snapshot_id": {"type": "string"},
|
||||
"scopes": {"type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
"required": ["from_snapshot_id", "to_snapshot_id"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_export_config",
|
||||
"description": "Export snapshot config. Defaults to summary-only; full=true returns redacted + size-capped data.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"snapshot_id": {"type": "string"},
|
||||
"full": {"type": "boolean", "default": False},
|
||||
"scopes": {"type": "array", "items": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_tunnel_status",
|
||||
"description": "Return tunnel status summary (connector count, last seen).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"snapshot_id": {"type": "string"},
|
||||
"tunnel_name": {"type": "string"},
|
||||
"tunnel_id": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_tunnel_ingress_summary",
|
||||
"description": "Parse cloudflared ingress hostnames from a local config file (never dumps full YAML unless full=true, still capped).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config_path": {
|
||||
"type": "string",
|
||||
"default": "/etc/cloudflared/config.yml",
|
||||
},
|
||||
"full": {"type": "boolean", "default": False},
|
||||
"max_rules": {"type": "integer", "default": 50},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "cf_access_policy_list",
|
||||
"description": "List Access apps, or policies for a specific app_id (summary-only).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"app_id": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class StdioJsonRpc:
|
||||
def __init__(self) -> None:
|
||||
self._in = sys.stdin.buffer
|
||||
self._out = sys.stdout.buffer
|
||||
self._mode: str | None = None # "headers" | "line"
|
||||
|
||||
def read_message(self) -> Optional[Dict[str, Any]]:
|
||||
while True:
|
||||
if self._mode == "line":
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
raw = line.decode("utf-8", "replace").strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
continue
|
||||
|
||||
first = self._in.readline()
|
||||
if not first:
|
||||
return None
|
||||
|
||||
if first in (b"\r\n", b"\n"):
|
||||
continue
|
||||
|
||||
# Auto-detect newline-delimited JSON framing.
|
||||
if self._mode is None and first.lstrip().startswith(b"{"):
|
||||
try:
|
||||
msg = json.loads(first.decode("utf-8", "replace"))
|
||||
except Exception:
|
||||
msg = None
|
||||
if isinstance(msg, dict):
|
||||
self._mode = "line"
|
||||
return msg
|
||||
|
||||
headers: Dict[str, str] = {}
|
||||
try:
|
||||
text = first.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
while True:
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
if line in (b"\r\n", b"\n"):
|
||||
break
|
||||
try:
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
if "content-length" not in headers:
|
||||
return None
|
||||
try:
|
||||
length = int(headers["content-length"])
|
||||
except ValueError:
|
||||
return None
|
||||
body = self._in.read(length)
|
||||
if not body:
|
||||
return None
|
||||
self._mode = "headers"
|
||||
msg = json.loads(body.decode("utf-8", "replace"))
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
return None
|
||||
|
||||
def write_message(self, message: Dict[str, Any]) -> None:
|
||||
if self._mode == "line":
|
||||
payload = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
self._out.write(payload + b"\n")
|
||||
self._out.flush()
|
||||
return
|
||||
|
||||
body = json.dumps(message, ensure_ascii=False, separators=(",", ":")).encode(
|
||||
"utf-8"
|
||||
)
|
||||
header = f"Content-Length: {len(body)}\r\n\r\n".encode("utf-8")
|
||||
self._out.write(header)
|
||||
self._out.write(body)
|
||||
self._out.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tools = CloudflareSafeTools()
|
||||
rpc = StdioJsonRpc()
|
||||
|
||||
handlers: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {
|
||||
"cf_snapshot": lambda a: tools.cf_snapshot(**a),
|
||||
"cf_refresh": lambda a: tools.cf_refresh(**a),
|
||||
"cf_config_diff": lambda a: tools.cf_config_diff(**a),
|
||||
"cf_export_config": lambda a: tools.cf_export_config(**a),
|
||||
"cf_tunnel_status": lambda a: tools.cf_tunnel_status(**a),
|
||||
"cf_tunnel_ingress_summary": lambda a: tools.cf_tunnel_ingress_summary(**a),
|
||||
"cf_access_policy_list": lambda a: tools.cf_access_policy_list(**a),
|
||||
}
|
||||
|
||||
while True:
|
||||
msg = rpc.read_message()
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
method = msg.get("method")
|
||||
msg_id = msg.get("id")
|
||||
params = msg.get("params") or {}
|
||||
|
||||
try:
|
||||
if method == "initialize":
|
||||
result = {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {"name": "cloudflare_safe", "version": "0.1.0"},
|
||||
"capabilities": {"tools": {}},
|
||||
}
|
||||
rpc.write_message({"jsonrpc": "2.0", "id": msg_id, "result": result})
|
||||
continue
|
||||
|
||||
if method == "tools/list":
|
||||
rpc.write_message(
|
||||
{"jsonrpc": "2.0", "id": msg_id, "result": {"tools": TOOLS}}
|
||||
)
|
||||
continue
|
||||
|
||||
if method == "tools/call":
|
||||
tool_name = str(params.get("name") or "")
|
||||
args = params.get("arguments") or {}
|
||||
if tool_name not in handlers:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"Unknown tool: {tool_name}",
|
||||
"data": {"known_tools": sorted(handlers.keys())},
|
||||
"truncated": False,
|
||||
"next_steps": ["Call tools/list"],
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = handlers[tool_name](args)
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(payload),
|
||||
}
|
||||
)
|
||||
except CloudflareError as e:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": str(e),
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Verify CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID are set",
|
||||
"Retry with a narrower scope",
|
||||
],
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"Unhandled error: {e}",
|
||||
"truncated": False,
|
||||
"next_steps": ["Retry with a narrower scope"],
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Ignore notifications.
|
||||
if msg_id is None:
|
||||
continue
|
||||
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"Unsupported method: {method}",
|
||||
"truncated": False,
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Last-resort: avoid crashing the server.
|
||||
if msg_id is not None:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"fatal error: {e}",
|
||||
"truncated": False,
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
6
mcp/oracle_answer/__main__.py
Normal file
6
mcp/oracle_answer/__main__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
386
mcp/oracle_answer/server.py
Normal file
386
mcp/oracle_answer/server.py
Normal file
@@ -0,0 +1,386 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from layer0 import layer0_entry
|
||||
from layer0.shadow_classifier import ShadowEvalResult
|
||||
|
||||
from .tool import OracleAnswerTool
|
||||
|
||||
MAX_BYTES_DEFAULT = 32_000
|
||||
|
||||
|
||||
def _max_bytes() -> int:
|
||||
raw = (os.getenv("VM_MCP_MAX_BYTES") or "").strip()
|
||||
if not raw:
|
||||
return MAX_BYTES_DEFAULT
|
||||
try:
|
||||
return max(4_096, int(raw))
|
||||
except ValueError:
|
||||
return MAX_BYTES_DEFAULT
|
||||
|
||||
|
||||
def _redact(obj: Any) -> Any:
|
||||
sensitive_keys = ("token", "secret", "password", "private", "key", "certificate")
|
||||
|
||||
if isinstance(obj, dict):
|
||||
out: Dict[str, Any] = {}
|
||||
for k, v in obj.items():
|
||||
if any(s in str(k).lower() for s in sensitive_keys):
|
||||
out[k] = "<REDACTED>"
|
||||
else:
|
||||
out[k] = _redact(v)
|
||||
return out
|
||||
if isinstance(obj, list):
|
||||
return [_redact(v) for v in obj]
|
||||
if isinstance(obj, str):
|
||||
if obj.startswith("ghp_") or obj.startswith("github_pat_"):
|
||||
return "<REDACTED>"
|
||||
return obj
|
||||
return obj
|
||||
|
||||
|
||||
def _safe_json(payload: Dict[str, Any]) -> str:
|
||||
payload = _redact(payload)
|
||||
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
if len(raw.encode("utf-8")) <= _max_bytes():
|
||||
return json.dumps(payload, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
truncated = {
|
||||
"ok": payload.get("ok", True),
|
||||
"truncated": True,
|
||||
"summary": payload.get("summary", "Response exceeded max size; truncated."),
|
||||
"next_steps": payload.get(
|
||||
"next_steps",
|
||||
["request narrower outputs (e.g., fewer frameworks or shorter question)"],
|
||||
),
|
||||
}
|
||||
return json.dumps(truncated, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
|
||||
def _mcp_text_result(
|
||||
payload: Dict[str, Any], *, is_error: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": [{"type": "text", "text": _safe_json(payload)}]
|
||||
}
|
||||
if is_error:
|
||||
result["isError"] = True
|
||||
return result
|
||||
|
||||
|
||||
TOOLS: List[Dict[str, Any]] = [
|
||||
{
|
||||
"name": "oracle_answer",
|
||||
"description": "Answer a compliance/security question (optionally via NVIDIA LLM) and map to frameworks.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "The question to answer.",
|
||||
},
|
||||
"frameworks": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Frameworks to reference (e.g., ['NIST-CSF','ISO-27001','GDPR']).",
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["strict", "advisory"],
|
||||
"default": "strict",
|
||||
"description": "strict=conservative, advisory=exploratory.",
|
||||
},
|
||||
"local_only": {
|
||||
"type": "boolean",
|
||||
"description": "If true, skip NVIDIA API calls (uses local-only mode). Defaults to true when NVIDIA_API_KEY is missing.",
|
||||
},
|
||||
},
|
||||
"required": ["question"],
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class OracleAnswerTools:
|
||||
async def oracle_answer(
|
||||
self,
|
||||
*,
|
||||
question: str,
|
||||
frameworks: Optional[List[str]] = None,
|
||||
mode: str = "strict",
|
||||
local_only: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
routing_action, shadow = layer0_entry(question)
|
||||
if routing_action != "HANDOFF_TO_LAYER1":
|
||||
return _layer0_payload(routing_action, shadow)
|
||||
|
||||
local_only_use = (
|
||||
bool(local_only)
|
||||
if local_only is not None
|
||||
else not bool((os.getenv("NVIDIA_API_KEY") or "").strip())
|
||||
)
|
||||
|
||||
try:
|
||||
tool = OracleAnswerTool(
|
||||
default_frameworks=frameworks,
|
||||
use_local_only=local_only_use,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": str(e),
|
||||
"data": {
|
||||
"local_only": local_only_use,
|
||||
"has_nvidia_api_key": bool(
|
||||
(os.getenv("NVIDIA_API_KEY") or "").strip()
|
||||
),
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Set NVIDIA_API_KEY to enable live answers",
|
||||
"Or call oracle_answer(local_only=true, ...)",
|
||||
],
|
||||
}
|
||||
|
||||
resp = await tool.answer(question=question, frameworks=frameworks, mode=mode)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "Oracle answer generated.",
|
||||
"data": {
|
||||
"question": question,
|
||||
"mode": mode,
|
||||
"frameworks": frameworks or tool.default_frameworks,
|
||||
"local_only": local_only_use,
|
||||
"model": resp.model,
|
||||
"answer": resp.answer,
|
||||
"framework_hits": resp.framework_hits,
|
||||
"reasoning": resp.reasoning,
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"If the answer is incomplete, add more specifics to the question or include more frameworks.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class StdioJsonRpc:
|
||||
def __init__(self) -> None:
|
||||
self._in = sys.stdin.buffer
|
||||
self._out = sys.stdout.buffer
|
||||
self._mode: str | None = None # "headers" | "line"
|
||||
|
||||
def read_message(self) -> Optional[Dict[str, Any]]:
|
||||
while True:
|
||||
if self._mode == "line":
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
raw = line.decode("utf-8", "replace").strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
continue
|
||||
|
||||
first = self._in.readline()
|
||||
if not first:
|
||||
return None
|
||||
|
||||
if first in (b"\r\n", b"\n"):
|
||||
continue
|
||||
|
||||
# Auto-detect newline-delimited JSON framing.
|
||||
if self._mode is None and first.lstrip().startswith(b"{"):
|
||||
try:
|
||||
msg = json.loads(first.decode("utf-8", "replace"))
|
||||
except Exception:
|
||||
msg = None
|
||||
if isinstance(msg, dict):
|
||||
self._mode = "line"
|
||||
return msg
|
||||
|
||||
headers: Dict[str, str] = {}
|
||||
try:
|
||||
text = first.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
while True:
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
if line in (b"\r\n", b"\n"):
|
||||
break
|
||||
try:
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
if "content-length" not in headers:
|
||||
return None
|
||||
try:
|
||||
length = int(headers["content-length"])
|
||||
except ValueError:
|
||||
return None
|
||||
body = self._in.read(length)
|
||||
if not body:
|
||||
return None
|
||||
self._mode = "headers"
|
||||
msg = json.loads(body.decode("utf-8", "replace"))
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
return None
|
||||
|
||||
def write_message(self, message: Dict[str, Any]) -> None:
|
||||
if self._mode == "line":
|
||||
payload = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
self._out.write(payload + b"\n")
|
||||
self._out.flush()
|
||||
return
|
||||
|
||||
body = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
header = f"Content-Length: {len(body)}\r\n\r\n".encode("utf-8")
|
||||
self._out.write(header)
|
||||
self._out.write(body)
|
||||
self._out.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tools = OracleAnswerTools()
|
||||
rpc = StdioJsonRpc()
|
||||
|
||||
handlers: Dict[str, Callable[[Dict[str, Any]], Any]] = {
|
||||
"oracle_answer": lambda a: tools.oracle_answer(**a),
|
||||
}
|
||||
|
||||
while True:
|
||||
msg = rpc.read_message()
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
method = msg.get("method")
|
||||
msg_id = msg.get("id")
|
||||
params = msg.get("params") or {}
|
||||
|
||||
try:
|
||||
if method == "initialize":
|
||||
result = {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {"name": "oracle_answer", "version": "0.1.0"},
|
||||
"capabilities": {"tools": {}},
|
||||
}
|
||||
rpc.write_message({"jsonrpc": "2.0", "id": msg_id, "result": result})
|
||||
continue
|
||||
|
||||
if method == "tools/list":
|
||||
rpc.write_message(
|
||||
{"jsonrpc": "2.0", "id": msg_id, "result": {"tools": TOOLS}}
|
||||
)
|
||||
continue
|
||||
|
||||
if method == "tools/call":
|
||||
tool_name = str(params.get("name") or "")
|
||||
args = params.get("arguments") or {}
|
||||
handler = handlers.get(tool_name)
|
||||
if not handler:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"Unknown tool: {tool_name}",
|
||||
"data": {"known_tools": sorted(handlers.keys())},
|
||||
"truncated": False,
|
||||
"next_steps": ["Call tools/list"],
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
payload = asyncio.run(handler(args)) # type: ignore[arg-type]
|
||||
is_error = (
|
||||
not bool(payload.get("ok", True))
|
||||
if isinstance(payload, dict)
|
||||
else False
|
||||
)
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(payload, is_error=is_error),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Ignore notifications.
|
||||
if msg_id is None:
|
||||
continue
|
||||
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{"ok": False, "summary": f"Unsupported method: {method}"},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if msg_id is not None:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{"ok": False, "summary": f"fatal error: {e}"},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _layer0_payload(routing_action: str, shadow: ShadowEvalResult) -> Dict[str, Any]:
|
||||
if routing_action == "FAIL_CLOSED":
|
||||
return {"ok": False, "summary": "Layer 0: cannot comply with this request."}
|
||||
if routing_action == "HANDOFF_TO_GUARDRAILS":
|
||||
reason = shadow.reason or "governance_violation"
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": f"Layer 0: governance violation detected ({reason}).",
|
||||
}
|
||||
if routing_action == "PROMPT_FOR_CLARIFICATION":
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": "Layer 0: request is ambiguous. Please clarify and retry.",
|
||||
}
|
||||
return {"ok": False, "summary": "Layer 0: unrecognized routing action; refusing."}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -9,7 +9,11 @@ Separate from CLI/API wrapper for clean testability.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -92,12 +96,10 @@ class OracleAnswerTool:
|
||||
if self.use_local_only:
|
||||
return "Local-only mode: skipping NVIDIA API call"
|
||||
|
||||
if not httpx:
|
||||
raise ImportError("httpx not installed. Install with: pip install httpx")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
payload = {
|
||||
@@ -108,18 +110,45 @@ class OracleAnswerTool:
|
||||
"max_tokens": 1024,
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
f"{self.NVIDIA_API_BASE}/chat/completions",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=30.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
# Prefer httpx when available; otherwise fall back to stdlib urllib to avoid extra deps.
|
||||
if httpx:
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
f"{self.NVIDIA_API_BASE}/chat/completions",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=30.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data["choices"][0]["message"]["content"]
|
||||
except Exception as e: # noqa: BLE001
|
||||
return f"(API Error: {str(e)}) Falling back to local analysis..."
|
||||
|
||||
def _urllib_post() -> str:
|
||||
req = urllib.request.Request(
|
||||
url=f"{self.NVIDIA_API_BASE}/chat/completions",
|
||||
method="POST",
|
||||
headers=headers,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read().decode("utf-8", "replace")
|
||||
data = json.loads(raw)
|
||||
return data["choices"][0]["message"]["content"]
|
||||
except Exception as e:
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = ""
|
||||
try:
|
||||
detail = e.read().decode("utf-8", "replace")
|
||||
except Exception:
|
||||
detail = str(e)
|
||||
raise RuntimeError(f"HTTP {e.code}: {detail}") from e
|
||||
|
||||
try:
|
||||
return await asyncio.to_thread(_urllib_post)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return f"(API Error: {str(e)}) Falling back to local analysis..."
|
||||
|
||||
async def answer(
|
||||
|
||||
@@ -10,22 +10,24 @@ This module provides tools to:
|
||||
Export primary classes and functions:
|
||||
"""
|
||||
|
||||
from mcp.waf_intelligence.analyzer import (
|
||||
WAFRuleAnalyzer,
|
||||
RuleViolation,
|
||||
__version__ = "0.3.0"
|
||||
|
||||
from .analyzer import (
|
||||
AnalysisResult,
|
||||
RuleViolation,
|
||||
WAFRuleAnalyzer,
|
||||
)
|
||||
from mcp.waf_intelligence.generator import (
|
||||
WAFRuleGenerator,
|
||||
GeneratedRule,
|
||||
)
|
||||
from mcp.waf_intelligence.compliance import (
|
||||
from .compliance import (
|
||||
ComplianceMapper,
|
||||
FrameworkMapping,
|
||||
)
|
||||
from mcp.waf_intelligence.orchestrator import (
|
||||
WAFIntelligence,
|
||||
from .generator import (
|
||||
GeneratedRule,
|
||||
WAFRuleGenerator,
|
||||
)
|
||||
from .orchestrator import (
|
||||
WAFInsight,
|
||||
WAFIntelligence,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
|
||||
from layer0 import layer0_entry
|
||||
from layer0.shadow_classifier import ShadowEvalResult
|
||||
|
||||
from . import __version__ as WAF_INTEL_VERSION
|
||||
from .orchestrator import WAFInsight, WAFIntelligence
|
||||
|
||||
|
||||
@@ -56,11 +57,18 @@ def run_cli(argv: List[str] | None = None) -> int:
|
||||
action="store_true",
|
||||
help="Exit with non-zero code if any error-severity violations are found.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
version=f"%(prog)s {WAF_INTEL_VERSION}",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# Layer 0: pre-boot Shadow Eval gate.
|
||||
routing_action, shadow = layer0_entry(f"waf_intel_cli file={args.file} limit={args.limit}")
|
||||
routing_action, shadow = layer0_entry(
|
||||
f"waf_intel_cli file={args.file} limit={args.limit}"
|
||||
)
|
||||
if routing_action != "HANDOFF_TO_LAYER1":
|
||||
_render_layer0_block(routing_action, shadow)
|
||||
return 1
|
||||
@@ -90,7 +98,9 @@ def run_cli(argv: List[str] | None = None) -> int:
|
||||
print(f"\nWAF Intelligence Report for: {path}\n{'-' * 72}")
|
||||
|
||||
if not insights:
|
||||
print("No high-severity, high-confidence issues detected based on current heuristics.")
|
||||
print(
|
||||
"No high-severity, high-confidence issues detected based on current heuristics."
|
||||
)
|
||||
return 0
|
||||
|
||||
for idx, insight in enumerate(insights, start=1):
|
||||
@@ -119,7 +129,9 @@ def run_cli(argv: List[str] | None = None) -> int:
|
||||
if insight.mappings:
|
||||
print("\nCompliance Mapping:")
|
||||
for mapping in insight.mappings:
|
||||
print(f" - {mapping.framework} {mapping.control_id}: {mapping.description}")
|
||||
print(
|
||||
f" - {mapping.framework} {mapping.control_id}: {mapping.description}"
|
||||
)
|
||||
|
||||
print()
|
||||
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
MANAGED_WAF_RULESET_IDS = (
|
||||
# Cloudflare managed WAF ruleset IDs (last updated 2025-12-18).
|
||||
"efb7b8c949ac4650a09736fc376e9aee", # Cloudflare Managed Ruleset
|
||||
"4814384a9e5d4991b9815dcfc25d2f1f", # OWASP Core Ruleset
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuleViolation:
|
||||
@@ -57,6 +64,20 @@ class WAFRuleAnalyzer:
|
||||
Analyze Cloudflare WAF rules from Terraform with a quality-first posture.
|
||||
"""
|
||||
|
||||
def _has_managed_waf_rules(self, text: str) -> bool:
|
||||
text_lower = text.lower()
|
||||
|
||||
if "managed_rules" in text_lower:
|
||||
return True
|
||||
|
||||
if re.search(r'phase\s*=\s*"http_request_firewall_managed"', text_lower):
|
||||
return True
|
||||
|
||||
if "cf.waf" in text_lower:
|
||||
return True
|
||||
|
||||
return any(ruleset_id in text_lower for ruleset_id in MANAGED_WAF_RULESET_IDS)
|
||||
|
||||
def analyze_file(
|
||||
self,
|
||||
path: str | Path,
|
||||
@@ -70,7 +91,7 @@ class WAFRuleAnalyzer:
|
||||
violations: List[RuleViolation] = []
|
||||
|
||||
# Example heuristic: no managed rules present
|
||||
if "managed_rules" not in text:
|
||||
if not self._has_managed_waf_rules(text):
|
||||
violations.append(
|
||||
RuleViolation(
|
||||
rule_id=None,
|
||||
@@ -102,7 +123,7 @@ class WAFRuleAnalyzer:
|
||||
violations=violations,
|
||||
metadata={
|
||||
"file_size": path.stat().st_size,
|
||||
"heuristics_version": "0.2.0",
|
||||
"heuristics_version": "0.3.0",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -125,7 +146,7 @@ class WAFRuleAnalyzer:
|
||||
tmp_path = Path(source_name)
|
||||
violations: List[RuleViolation] = []
|
||||
|
||||
if "managed_rules" not in text:
|
||||
if not self._has_managed_waf_rules(text):
|
||||
violations.append(
|
||||
RuleViolation(
|
||||
rule_id=None,
|
||||
@@ -141,7 +162,7 @@ class WAFRuleAnalyzer:
|
||||
result = AnalysisResult(
|
||||
source=str(tmp_path),
|
||||
violations=violations,
|
||||
metadata={"heuristics_version": "0.2.0"},
|
||||
metadata={"heuristics_version": "0.3.0"},
|
||||
)
|
||||
|
||||
result.violations = result.top_violations(
|
||||
@@ -172,15 +193,25 @@ class WAFRuleAnalyzer:
|
||||
AnalysisResult with violations informed by threat intel
|
||||
"""
|
||||
# Start with base analysis
|
||||
base_result = self.analyze_file(path, min_severity=min_severity, min_confidence=min_confidence)
|
||||
base_result = self.analyze_file(
|
||||
path, min_severity=min_severity, min_confidence=min_confidence
|
||||
)
|
||||
|
||||
path = Path(path)
|
||||
text = path.read_text(encoding="utf-8")
|
||||
text_lower = text.lower()
|
||||
|
||||
# Check if threat indicators are addressed by existing rules
|
||||
critical_ips = [i for i in threat_indicators if i.indicator_type == "ip" and i.severity in ("critical", "high")]
|
||||
critical_patterns = [i for i in threat_indicators if i.indicator_type == "pattern" and i.severity in ("critical", "high")]
|
||||
critical_ips = [
|
||||
i
|
||||
for i in threat_indicators
|
||||
if i.indicator_type == "ip" and i.severity in ("critical", "high")
|
||||
]
|
||||
critical_patterns = [
|
||||
i
|
||||
for i in threat_indicators
|
||||
if i.indicator_type == "pattern" and i.severity in ("critical", "high")
|
||||
]
|
||||
|
||||
# Check for IP blocking coverage
|
||||
if critical_ips:
|
||||
@@ -228,4 +259,3 @@ class WAFRuleAnalyzer:
|
||||
}
|
||||
|
||||
return base_result
|
||||
|
||||
|
||||
632
mcp/waf_intelligence/mcp_server.py
Normal file
632
mcp/waf_intelligence/mcp_server.py
Normal file
@@ -0,0 +1,632 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from cloudflare.layer0 import layer0_entry
|
||||
from cloudflare.layer0.shadow_classifier import ShadowEvalResult
|
||||
|
||||
from .orchestrator import ThreatAssessment, WAFInsight, WAFIntelligence
|
||||
|
||||
MAX_BYTES_DEFAULT = 32_000
|
||||
|
||||
|
||||
def _cloudflare_root() -> Path:
|
||||
# mcp_server.py -> waf_intelligence -> mcp -> cloudflare
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _max_bytes() -> int:
|
||||
raw = (os.getenv("VM_MCP_MAX_BYTES") or "").strip()
|
||||
if not raw:
|
||||
return MAX_BYTES_DEFAULT
|
||||
try:
|
||||
return max(4_096, int(raw))
|
||||
except ValueError:
|
||||
return MAX_BYTES_DEFAULT
|
||||
|
||||
|
||||
def _redact(obj: Any) -> Any:
|
||||
sensitive_keys = ("token", "secret", "password", "private", "key", "certificate")
|
||||
|
||||
if isinstance(obj, dict):
|
||||
out: Dict[str, Any] = {}
|
||||
for k, v in obj.items():
|
||||
if any(s in str(k).lower() for s in sensitive_keys):
|
||||
out[k] = "<REDACTED>"
|
||||
else:
|
||||
out[k] = _redact(v)
|
||||
return out
|
||||
if isinstance(obj, list):
|
||||
return [_redact(v) for v in obj]
|
||||
if isinstance(obj, str):
|
||||
if obj.startswith("ghp_") or obj.startswith("github_pat_"):
|
||||
return "<REDACTED>"
|
||||
return obj
|
||||
return obj
|
||||
|
||||
|
||||
def _safe_json(payload: Dict[str, Any]) -> str:
|
||||
payload = _redact(payload)
|
||||
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
if len(raw.encode("utf-8")) <= _max_bytes():
|
||||
return json.dumps(payload, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
truncated = {
|
||||
"ok": payload.get("ok", True),
|
||||
"truncated": True,
|
||||
"summary": payload.get("summary", "Response exceeded max size; truncated."),
|
||||
"next_steps": payload.get(
|
||||
"next_steps",
|
||||
[
|
||||
"request fewer files/insights (limit=...)",
|
||||
"use higher min_severity to reduce output",
|
||||
],
|
||||
),
|
||||
}
|
||||
return json.dumps(truncated, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
|
||||
def _mcp_text_result(
|
||||
payload: Dict[str, Any], *, is_error: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {
|
||||
"content": [{"type": "text", "text": _safe_json(payload)}]
|
||||
}
|
||||
if is_error:
|
||||
result["isError"] = True
|
||||
return result
|
||||
|
||||
|
||||
def _insight_to_dict(insight: WAFInsight) -> Dict[str, Any]:
|
||||
return asdict(insight)
|
||||
|
||||
|
||||
def _assessment_to_dict(assessment: ThreatAssessment) -> Dict[str, Any]:
|
||||
violations = []
|
||||
if assessment.analysis_result and getattr(
|
||||
assessment.analysis_result, "violations", None
|
||||
):
|
||||
violations = list(assessment.analysis_result.violations)
|
||||
|
||||
severity_counts = {"error": 0, "warning": 0, "info": 0}
|
||||
for v in violations:
|
||||
sev = getattr(v, "severity", "info")
|
||||
if sev in severity_counts:
|
||||
severity_counts[sev] += 1
|
||||
|
||||
return {
|
||||
"risk_score": assessment.risk_score,
|
||||
"risk_level": assessment.risk_level,
|
||||
"classification_summary": assessment.classification_summary,
|
||||
"recommended_actions": assessment.recommended_actions,
|
||||
"analysis": {
|
||||
"has_config_analysis": assessment.analysis_result is not None,
|
||||
"violations_total": len(violations),
|
||||
"violations_by_severity": severity_counts,
|
||||
},
|
||||
"has_threat_intel": assessment.threat_report is not None,
|
||||
"generated_at": str(assessment.generated_at),
|
||||
}
|
||||
|
||||
|
||||
TOOLS: List[Dict[str, Any]] = [
|
||||
{
|
||||
"name": "waf_capabilities",
|
||||
"description": "List available WAF Intelligence capabilities.",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
{
|
||||
"name": "analyze_waf",
|
||||
"description": "Analyze Terraform WAF file(s) and return curated insights (legacy alias for waf_analyze).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {
|
||||
"type": "string",
|
||||
"description": "Single file path to analyze.",
|
||||
},
|
||||
"files": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of file paths or glob patterns to analyze.",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"default": 3,
|
||||
"description": "Max insights per file.",
|
||||
},
|
||||
"severity_threshold": {
|
||||
"type": "string",
|
||||
"enum": ["info", "warning", "error"],
|
||||
"default": "warning",
|
||||
"description": "Minimum severity to include (alias for min_severity).",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "waf_analyze",
|
||||
"description": "Analyze Terraform WAF file(s) and return curated insights (requires file or files).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {
|
||||
"type": "string",
|
||||
"description": "Single file path to analyze.",
|
||||
},
|
||||
"files": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of file paths or glob patterns to analyze.",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"default": 3,
|
||||
"description": "Max insights per file.",
|
||||
},
|
||||
"min_severity": {
|
||||
"type": "string",
|
||||
"enum": ["info", "warning", "error"],
|
||||
"default": "warning",
|
||||
"description": "Minimum severity to include.",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "waf_assess",
|
||||
"description": "Run a broader assessment (optionally includes threat intel collection).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"waf_config_path": {
|
||||
"type": "string",
|
||||
"description": "Path to Terraform WAF config (default: terraform/waf.tf).",
|
||||
},
|
||||
"include_threat_intel": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "If true, attempt to collect threat intel (may require network and credentials).",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "waf_generate_gitops_proposals",
|
||||
"description": "Generate GitOps-ready rule proposals (best-effort; requires threat intel to produce output).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"waf_config_path": {
|
||||
"type": "string",
|
||||
"description": "Path to Terraform WAF config (default: terraform/waf.tf).",
|
||||
},
|
||||
"include_threat_intel": {
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "Attempt to collect threat intel before proposing rules.",
|
||||
},
|
||||
"max_proposals": {
|
||||
"type": "integer",
|
||||
"default": 5,
|
||||
"description": "Maximum proposals to generate.",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class WafIntelligenceTools:
|
||||
def __init__(self) -> None:
|
||||
self.workspace_root = _cloudflare_root()
|
||||
self.repo_root = self.workspace_root.parent
|
||||
self.waf = WAFIntelligence(workspace_path=str(self.workspace_root))
|
||||
|
||||
def _resolve_path(self, raw: str) -> Path:
|
||||
path = Path(raw)
|
||||
if path.is_absolute():
|
||||
return path
|
||||
|
||||
candidates = [
|
||||
Path.cwd() / path,
|
||||
self.workspace_root / path,
|
||||
self.repo_root / path,
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return self.workspace_root / path
|
||||
|
||||
def waf_capabilities(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "WAF Intelligence capabilities.",
|
||||
"data": {"capabilities": self.waf.capabilities},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Call waf_analyze(file=..., limit=...) to analyze config.",
|
||||
"Call waf_assess(include_threat_intel=true) for a broader assessment.",
|
||||
],
|
||||
}
|
||||
|
||||
def waf_analyze(
|
||||
self,
|
||||
*,
|
||||
file: Optional[str] = None,
|
||||
files: Optional[List[str]] = None,
|
||||
limit: int = 3,
|
||||
min_severity: str = "warning",
|
||||
) -> Dict[str, Any]:
|
||||
paths: List[str] = []
|
||||
if files:
|
||||
for pattern in files:
|
||||
paths.extend(glob.glob(pattern))
|
||||
if file:
|
||||
paths.append(file)
|
||||
|
||||
seen = set()
|
||||
unique_paths: List[str] = []
|
||||
for p in paths:
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
unique_paths.append(p)
|
||||
|
||||
if not unique_paths:
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": "Provide 'file' or 'files' to analyze.",
|
||||
"truncated": False,
|
||||
"next_steps": ["Call waf_analyze(file='terraform/waf.tf')"],
|
||||
}
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
for p in unique_paths:
|
||||
path = self._resolve_path(p)
|
||||
if not path.exists():
|
||||
results.append(
|
||||
{
|
||||
"file": str(path),
|
||||
"ok": False,
|
||||
"summary": "File not found.",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
insights = self.waf.analyze_and_recommend(
|
||||
str(path),
|
||||
limit=limit,
|
||||
min_severity=min_severity,
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"file": str(path),
|
||||
"ok": True,
|
||||
"insights": [_insight_to_dict(i) for i in insights],
|
||||
}
|
||||
)
|
||||
|
||||
ok = all(r.get("ok") for r in results)
|
||||
return {
|
||||
"ok": ok,
|
||||
"summary": f"Analyzed {len(results)} file(s).",
|
||||
"data": {"results": results},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Raise/lower min_severity or limit to tune output size.",
|
||||
],
|
||||
}
|
||||
|
||||
def waf_assess(
|
||||
self,
|
||||
*,
|
||||
waf_config_path: Optional[str] = None,
|
||||
include_threat_intel: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
waf_config_path_resolved = (
|
||||
str(self._resolve_path(waf_config_path)) if waf_config_path else None
|
||||
)
|
||||
assessment = self.waf.full_assessment(
|
||||
waf_config_path=waf_config_path_resolved,
|
||||
include_threat_intel=include_threat_intel,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": "WAF assessment complete.",
|
||||
"data": _assessment_to_dict(assessment),
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"Call waf_generate_gitops_proposals(...) to draft Terraform rule proposals (best-effort).",
|
||||
],
|
||||
}
|
||||
|
||||
def waf_generate_gitops_proposals(
|
||||
self,
|
||||
*,
|
||||
waf_config_path: Optional[str] = None,
|
||||
include_threat_intel: bool = True,
|
||||
max_proposals: int = 5,
|
||||
) -> Dict[str, Any]:
|
||||
waf_config_path_resolved = (
|
||||
str(self._resolve_path(waf_config_path)) if waf_config_path else None
|
||||
)
|
||||
assessment = self.waf.full_assessment(
|
||||
waf_config_path=waf_config_path_resolved,
|
||||
include_threat_intel=include_threat_intel,
|
||||
)
|
||||
proposals = self.waf.generate_gitops_proposals(
|
||||
threat_report=assessment.threat_report,
|
||||
max_proposals=max_proposals,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"summary": f"Generated {len(proposals)} proposal(s).",
|
||||
"data": {
|
||||
"assessment": _assessment_to_dict(assessment),
|
||||
"proposals": proposals,
|
||||
},
|
||||
"truncated": False,
|
||||
"next_steps": [
|
||||
"If proposals are empty, enable threat intel and ensure required credentials/log sources exist.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class StdioJsonRpc:
|
||||
def __init__(self) -> None:
|
||||
self._in = sys.stdin.buffer
|
||||
self._out = sys.stdout.buffer
|
||||
self._mode: str | None = None # "headers" | "line"
|
||||
|
||||
def read_message(self) -> Optional[Dict[str, Any]]:
|
||||
while True:
|
||||
if self._mode == "line":
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
raw = line.decode("utf-8", "replace").strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
continue
|
||||
|
||||
first = self._in.readline()
|
||||
if not first:
|
||||
return None
|
||||
|
||||
if first in (b"\r\n", b"\n"):
|
||||
continue
|
||||
|
||||
# Auto-detect newline-delimited JSON framing.
|
||||
if self._mode is None and first.lstrip().startswith(b"{"):
|
||||
try:
|
||||
msg = json.loads(first.decode("utf-8", "replace"))
|
||||
except Exception:
|
||||
msg = None
|
||||
if isinstance(msg, dict):
|
||||
self._mode = "line"
|
||||
return msg
|
||||
|
||||
headers: Dict[str, str] = {}
|
||||
try:
|
||||
text = first.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
while True:
|
||||
line = self._in.readline()
|
||||
if not line:
|
||||
return None
|
||||
if line in (b"\r\n", b"\n"):
|
||||
break
|
||||
try:
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if ":" not in text:
|
||||
continue
|
||||
k, v = text.split(":", 1)
|
||||
headers[k.lower().strip()] = v.strip()
|
||||
|
||||
if "content-length" not in headers:
|
||||
return None
|
||||
try:
|
||||
length = int(headers["content-length"])
|
||||
except ValueError:
|
||||
return None
|
||||
body = self._in.read(length)
|
||||
if not body:
|
||||
return None
|
||||
self._mode = "headers"
|
||||
msg = json.loads(body.decode("utf-8", "replace"))
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
return None
|
||||
|
||||
def write_message(self, message: Dict[str, Any]) -> None:
|
||||
if self._mode == "line":
|
||||
payload = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
self._out.write(payload + b"\n")
|
||||
self._out.flush()
|
||||
return
|
||||
|
||||
body = json.dumps(
|
||||
message, ensure_ascii=False, separators=(",", ":"), default=str
|
||||
).encode("utf-8")
|
||||
header = f"Content-Length: {len(body)}\r\n\r\n".encode("utf-8")
|
||||
self._out.write(header)
|
||||
self._out.write(body)
|
||||
self._out.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tools = WafIntelligenceTools()
|
||||
rpc = StdioJsonRpc()
|
||||
|
||||
handlers: Dict[str, Callable[[Dict[str, Any]], Dict[str, Any]]] = {
|
||||
"waf_capabilities": lambda a: tools.waf_capabilities(),
|
||||
"analyze_waf": lambda a: tools.waf_analyze(
|
||||
file=a.get("file"),
|
||||
files=a.get("files"),
|
||||
limit=int(a.get("limit", 3)),
|
||||
min_severity=str(a.get("severity_threshold", "warning")),
|
||||
),
|
||||
"waf_analyze": lambda a: tools.waf_analyze(**a),
|
||||
"waf_assess": lambda a: tools.waf_assess(**a),
|
||||
"waf_generate_gitops_proposals": lambda a: tools.waf_generate_gitops_proposals(
|
||||
**a
|
||||
),
|
||||
}
|
||||
|
||||
while True:
|
||||
msg = rpc.read_message()
|
||||
if msg is None:
|
||||
return
|
||||
|
||||
method = msg.get("method")
|
||||
msg_id = msg.get("id")
|
||||
params = msg.get("params") or {}
|
||||
|
||||
try:
|
||||
if method == "initialize":
|
||||
result = {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {"name": "waf_intelligence", "version": "0.1.0"},
|
||||
"capabilities": {"tools": {}},
|
||||
}
|
||||
rpc.write_message({"jsonrpc": "2.0", "id": msg_id, "result": result})
|
||||
continue
|
||||
|
||||
if method == "tools/list":
|
||||
rpc.write_message(
|
||||
{"jsonrpc": "2.0", "id": msg_id, "result": {"tools": TOOLS}}
|
||||
)
|
||||
continue
|
||||
|
||||
if method == "tools/call":
|
||||
tool_name = str(params.get("name") or "")
|
||||
args = params.get("arguments") or {}
|
||||
|
||||
routing_action, shadow = layer0_entry(
|
||||
_shadow_query_repr(tool_name, args)
|
||||
)
|
||||
if routing_action != "HANDOFF_TO_LAYER1":
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
_layer0_payload(routing_action, shadow), is_error=True
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
handler = handlers.get(tool_name)
|
||||
if not handler:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{
|
||||
"ok": False,
|
||||
"summary": f"Unknown tool: {tool_name}",
|
||||
"data": {"known_tools": sorted(handlers.keys())},
|
||||
"truncated": False,
|
||||
"next_steps": ["Call tools/list"],
|
||||
},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
payload = handler(args)
|
||||
is_error = (
|
||||
not bool(payload.get("ok", True))
|
||||
if isinstance(payload, dict)
|
||||
else False
|
||||
)
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(payload, is_error=is_error),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Ignore notifications.
|
||||
if msg_id is None:
|
||||
continue
|
||||
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{"ok": False, "summary": f"Unsupported method: {method}"},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if msg_id is not None:
|
||||
rpc.write_message(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"result": _mcp_text_result(
|
||||
{"ok": False, "summary": f"fatal error: {e}"},
|
||||
is_error=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _shadow_query_repr(tool_name: str, tool_args: Dict[str, Any]) -> str:
|
||||
if tool_name == "waf_capabilities":
|
||||
return "List WAF Intelligence capabilities."
|
||||
try:
|
||||
return f"{tool_name}: {json.dumps(tool_args, sort_keys=True, default=str)}"
|
||||
except Exception:
|
||||
return f"{tool_name}: {str(tool_args)}"
|
||||
|
||||
|
||||
def _layer0_payload(routing_action: str, shadow: ShadowEvalResult) -> Dict[str, Any]:
|
||||
if routing_action == "FAIL_CLOSED":
|
||||
return {"ok": False, "summary": "Layer 0: cannot comply with this request."}
|
||||
if routing_action == "HANDOFF_TO_GUARDRAILS":
|
||||
reason = shadow.reason or "governance_violation"
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": f"Layer 0: governance violation detected ({reason}).",
|
||||
}
|
||||
if routing_action == "PROMPT_FOR_CLARIFICATION":
|
||||
return {
|
||||
"ok": False,
|
||||
"summary": "Layer 0: request is ambiguous. Please clarify and retry.",
|
||||
}
|
||||
return {"ok": False, "summary": "Layer 0: unrecognized routing action; refusing."}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,27 +6,26 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from mcp.waf_intelligence.analyzer import AnalysisResult, RuleViolation, WAFRuleAnalyzer
|
||||
from mcp.waf_intelligence.compliance import ComplianceMapper, FrameworkMapping
|
||||
from mcp.waf_intelligence.generator import GeneratedRule, WAFRuleGenerator
|
||||
from .analyzer import AnalysisResult, RuleViolation, WAFRuleAnalyzer
|
||||
from .compliance import ComplianceMapper, FrameworkMapping
|
||||
from .generator import GeneratedRule, WAFRuleGenerator
|
||||
|
||||
# Optional advanced modules (Phase 7)
|
||||
try:
|
||||
from mcp.waf_intelligence.threat_intel import (
|
||||
from .threat_intel import (
|
||||
ThreatIntelCollector,
|
||||
ThreatIntelReport,
|
||||
ThreatIndicator,
|
||||
)
|
||||
|
||||
_HAS_THREAT_INTEL = True
|
||||
except ImportError:
|
||||
_HAS_THREAT_INTEL = False
|
||||
ThreatIntelCollector = None
|
||||
|
||||
try:
|
||||
from mcp.waf_intelligence.classifier import (
|
||||
ThreatClassifier,
|
||||
ClassificationResult,
|
||||
)
|
||||
from .classifier import ThreatClassifier
|
||||
|
||||
_HAS_CLASSIFIER = True
|
||||
except ImportError:
|
||||
_HAS_CLASSIFIER = False
|
||||
@@ -232,7 +231,8 @@ class WAFIntelligence:
|
||||
|
||||
# Generate recommendations
|
||||
critical_count = sum(
|
||||
1 for v in assessment.analysis_result.violations
|
||||
1
|
||||
for v in assessment.analysis_result.violations
|
||||
if v.severity == "error"
|
||||
)
|
||||
if critical_count > 0:
|
||||
@@ -258,7 +258,9 @@ class WAFIntelligence:
|
||||
severity_counts[sev] = severity_counts.get(sev, 0) + 1
|
||||
|
||||
# Add to classification summary
|
||||
assessment.classification_summary["threat_indicators"] = len(indicators)
|
||||
assessment.classification_summary["threat_indicators"] = len(
|
||||
indicators
|
||||
)
|
||||
assessment.classification_summary.update(severity_counts)
|
||||
|
||||
# Calculate threat intel risk
|
||||
@@ -281,7 +283,8 @@ class WAFIntelligence:
|
||||
|
||||
indicators = assessment.threat_report.indicators
|
||||
pattern_indicators = [
|
||||
i for i in indicators
|
||||
i
|
||||
for i in indicators
|
||||
if getattr(i, "indicator_type", "") == "pattern"
|
||||
]
|
||||
|
||||
@@ -305,7 +308,9 @@ class WAFIntelligence:
|
||||
|
||||
# 4. Calculate final risk score
|
||||
if risk_factors:
|
||||
assessment.risk_score = min(1.0, sum(risk_factors) / max(len(risk_factors), 1))
|
||||
assessment.risk_score = min(
|
||||
1.0, sum(risk_factors) / max(len(risk_factors), 1)
|
||||
)
|
||||
else:
|
||||
assessment.risk_score = 0.3 # Baseline risk
|
||||
|
||||
@@ -344,15 +349,17 @@ class WAFIntelligence:
|
||||
)
|
||||
|
||||
for proposal in batch.proposals:
|
||||
proposals.append({
|
||||
"name": proposal.rule_name,
|
||||
"type": proposal.rule_type,
|
||||
"severity": proposal.severity,
|
||||
"confidence": proposal.confidence,
|
||||
"terraform": proposal.terraform_code,
|
||||
"justification": proposal.justification,
|
||||
"auto_deploy": proposal.auto_deploy_eligible,
|
||||
})
|
||||
proposals.append(
|
||||
{
|
||||
"name": proposal.rule_name,
|
||||
"type": proposal.rule_type,
|
||||
"severity": proposal.severity,
|
||||
"confidence": proposal.confidence,
|
||||
"terraform": proposal.terraform_code,
|
||||
"justification": proposal.justification,
|
||||
"auto_deploy": proposal.auto_deploy_eligible,
|
||||
}
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
326
mcp/waf_intelligence/server.py
Executable file → Normal file
326
mcp/waf_intelligence/server.py
Executable file → Normal file
@@ -1,326 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
WAF Intelligence MCP Server for VS Code Copilot.
|
||||
from __future__ import annotations
|
||||
|
||||
This implements the Model Context Protocol (MCP) stdio interface
|
||||
so VS Code can communicate with your WAF Intelligence system.
|
||||
"""
|
||||
Deprecated entrypoint kept for older editor configs.
|
||||
|
||||
Use `python3 -m mcp.waf_intelligence.mcp_server` (or `waf_intel_mcp.py`) instead.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
# Add parent to path for imports
|
||||
sys.path.insert(0, '/Users/sovereign/Desktop/CLOUDFLARE')
|
||||
|
||||
from mcp.waf_intelligence.orchestrator import WAFIntelligence
|
||||
from mcp.waf_intelligence.analyzer import WAFRuleAnalyzer
|
||||
from layer0 import layer0_entry
|
||||
from layer0.shadow_classifier import ShadowEvalResult
|
||||
|
||||
|
||||
class WAFIntelligenceMCPServer:
|
||||
"""MCP Server wrapper for WAF Intelligence."""
|
||||
|
||||
def __init__(self):
|
||||
self.waf = WAFIntelligence()
|
||||
self.analyzer = WAFRuleAnalyzer()
|
||||
|
||||
def get_capabilities(self) -> dict:
|
||||
"""Return server capabilities."""
|
||||
return {
|
||||
"tools": [
|
||||
{
|
||||
"name": "waf_analyze",
|
||||
"description": "Analyze WAF logs and detect attack patterns",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"log_file": {
|
||||
"type": "string",
|
||||
"description": "Path to WAF log file (optional)"
|
||||
},
|
||||
"zone_id": {
|
||||
"type": "string",
|
||||
"description": "Cloudflare zone ID (optional)"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "waf_assess",
|
||||
"description": "Run full security assessment with threat intel and ML classification",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"zone_id": {
|
||||
"type": "string",
|
||||
"description": "Cloudflare zone ID"
|
||||
}
|
||||
},
|
||||
"required": ["zone_id"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "waf_generate_rules",
|
||||
"description": "Generate Terraform WAF rules from threat intelligence",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"zone_id": {
|
||||
"type": "string",
|
||||
"description": "Cloudflare zone ID"
|
||||
},
|
||||
"min_confidence": {
|
||||
"type": "number",
|
||||
"description": "Minimum confidence threshold (0-1)",
|
||||
"default": 0.7
|
||||
}
|
||||
},
|
||||
"required": ["zone_id"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "waf_capabilities",
|
||||
"description": "List available WAF Intelligence capabilities",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
def handle_tool_call(self, name: str, arguments: dict) -> dict:
|
||||
"""Handle a tool invocation."""
|
||||
try:
|
||||
if name == "waf_capabilities":
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": json.dumps({
|
||||
"capabilities": self.waf.capabilities,
|
||||
"status": "operational"
|
||||
}, indent=2)
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
elif name == "waf_analyze":
|
||||
log_file = arguments.get("log_file")
|
||||
zone_id = arguments.get("zone_id")
|
||||
|
||||
if log_file:
|
||||
result = self.analyzer.analyze_log_file(log_file)
|
||||
else:
|
||||
result = {
|
||||
"message": "No log file provided. Use zone_id for live analysis.",
|
||||
"capabilities": self.waf.capabilities
|
||||
}
|
||||
|
||||
return {
|
||||
"content": [
|
||||
{"type": "text", "text": json.dumps(result, indent=2, default=str)}
|
||||
]
|
||||
}
|
||||
|
||||
elif name == "waf_assess":
|
||||
zone_id = arguments.get("zone_id")
|
||||
# full_assessment uses workspace paths, not zone_id
|
||||
assessment = self.waf.full_assessment(
|
||||
include_threat_intel=True
|
||||
)
|
||||
# Build result from ThreatAssessment dataclass
|
||||
result = {
|
||||
"zone_id": zone_id,
|
||||
"risk_score": assessment.risk_score,
|
||||
"risk_level": assessment.risk_level,
|
||||
"classification_summary": assessment.classification_summary,
|
||||
"recommended_actions": assessment.recommended_actions[:10], # Top 10
|
||||
"has_analysis": assessment.analysis_result is not None,
|
||||
"has_threat_intel": assessment.threat_report is not None,
|
||||
"generated_at": str(assessment.generated_at)
|
||||
}
|
||||
|
||||
return {
|
||||
"content": [
|
||||
{"type": "text", "text": json.dumps(result, indent=2, default=str)}
|
||||
]
|
||||
}
|
||||
|
||||
elif name == "waf_generate_rules":
|
||||
zone_id = arguments.get("zone_id")
|
||||
min_confidence = arguments.get("min_confidence", 0.7)
|
||||
|
||||
# Generate proposals (doesn't use zone_id directly)
|
||||
proposals = self.waf.generate_gitops_proposals(
|
||||
max_proposals=5
|
||||
)
|
||||
|
||||
result = {
|
||||
"zone_id": zone_id,
|
||||
"min_confidence": min_confidence,
|
||||
"proposals_count": len(proposals),
|
||||
"proposals": proposals
|
||||
}
|
||||
|
||||
return {
|
||||
"content": [
|
||||
{"type": "text", "text": json.dumps(result, indent=2, default=str) if proposals else "No rules generated (no threat data available)"}
|
||||
]
|
||||
}
|
||||
|
||||
else:
|
||||
return {
|
||||
"content": [
|
||||
{"type": "text", "text": f"Unknown tool: {name}"}
|
||||
],
|
||||
"isError": True
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"content": [
|
||||
{"type": "text", "text": f"Error: {str(e)}"}
|
||||
],
|
||||
"isError": True
|
||||
}
|
||||
|
||||
def run(self):
|
||||
"""Run the MCP server (stdio mode)."""
|
||||
# Send server info
|
||||
server_info = {
|
||||
"jsonrpc": "2.0",
|
||||
"method": "initialized",
|
||||
"params": {
|
||||
"serverInfo": {
|
||||
"name": "waf-intelligence",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
"capabilities": self.get_capabilities()
|
||||
}
|
||||
}
|
||||
|
||||
# Main loop - read JSON-RPC messages from stdin
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
message = json.loads(line.strip())
|
||||
|
||||
if message.get("method") == "initialize":
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {
|
||||
"name": "waf-intelligence",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
elif message.get("method") == "tools/list":
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"result": self.get_capabilities()
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
elif message.get("method") == "tools/call":
|
||||
params = message.get("params", {})
|
||||
tool_name = params.get("name")
|
||||
tool_args = params.get("arguments", {})
|
||||
|
||||
# Layer 0: pre-boot Shadow Eval gate before handling tool calls.
|
||||
routing_action, shadow = layer0_entry(_shadow_query_repr(tool_name, tool_args))
|
||||
if routing_action != "HANDOFF_TO_LAYER1":
|
||||
response = _layer0_mcp_response(routing_action, shadow, message.get("id"))
|
||||
print(json.dumps(response), flush=True)
|
||||
continue
|
||||
|
||||
result = self.handle_tool_call(tool_name, tool_args)
|
||||
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"result": result
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
elif message.get("method") == "notifications/initialized":
|
||||
# Client acknowledged initialization
|
||||
pass
|
||||
|
||||
else:
|
||||
# Unknown method
|
||||
response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"error": {
|
||||
"code": -32601,
|
||||
"message": f"Method not found: {message.get('method')}"
|
||||
}
|
||||
}
|
||||
print(json.dumps(response), flush=True)
|
||||
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except Exception as e:
|
||||
error_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": None,
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": str(e)
|
||||
}
|
||||
}
|
||||
print(json.dumps(error_response), flush=True)
|
||||
|
||||
from .mcp_server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
server = WAFIntelligenceMCPServer()
|
||||
server.run()
|
||||
main()
|
||||
|
||||
|
||||
def _shadow_query_repr(tool_name: str, tool_args: dict) -> str:
|
||||
"""Build a textual representation of the tool call for Layer 0 classification."""
|
||||
try:
|
||||
return f"{tool_name}: {json.dumps(tool_args, sort_keys=True)}"
|
||||
except TypeError:
|
||||
return f"{tool_name}: {str(tool_args)}"
|
||||
|
||||
|
||||
def _layer0_mcp_response(routing_action: str, shadow: ShadowEvalResult, msg_id: Any) -> dict:
|
||||
"""
|
||||
Map Layer 0 outcomes to MCP responses.
|
||||
Catastrophic/forbidden/ambiguous short-circuit with minimal disclosure.
|
||||
"""
|
||||
base = {"jsonrpc": "2.0", "id": msg_id}
|
||||
|
||||
if routing_action == "FAIL_CLOSED":
|
||||
base["error"] = {"code": -32000, "message": "Layer 0: cannot comply with this request."}
|
||||
return base
|
||||
|
||||
if routing_action == "HANDOFF_TO_GUARDRAILS":
|
||||
reason = shadow.reason or "governance_violation"
|
||||
base["error"] = {
|
||||
"code": -32001,
|
||||
"message": f"Layer 0: governance violation detected ({reason}).",
|
||||
}
|
||||
return base
|
||||
|
||||
if routing_action == "PROMPT_FOR_CLARIFICATION":
|
||||
base["error"] = {
|
||||
"code": -32002,
|
||||
"message": "Layer 0: request is ambiguous. Please clarify and retry.",
|
||||
}
|
||||
return base
|
||||
|
||||
base["error"] = {"code": -32099, "message": "Layer 0: unrecognized routing action; refusing."}
|
||||
return base
|
||||
|
||||
@@ -6,18 +6,18 @@
|
||||
// File system operations
|
||||
"filesystem": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-filesystem"],
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-filesystem", "."],
|
||||
"environment": {
|
||||
"HOME": "{env:HOME}"
|
||||
"HOME": "{env:HOME}",
|
||||
},
|
||||
"enabled": true
|
||||
"enabled": true,
|
||||
},
|
||||
|
||||
// Git operations
|
||||
"git": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-git"],
|
||||
"enabled": true
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// GitHub integration
|
||||
@@ -25,9 +25,9 @@
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-github"],
|
||||
"environment": {
|
||||
"GITHUB_PERSONAL_ACCESS_TOKEN": "{env:GITHUB_TOKEN}"
|
||||
"GITHUB_PERSONAL_ACCESS_TOKEN": "{env:GITHUB_TOKEN}",
|
||||
},
|
||||
"enabled": true
|
||||
"enabled": true,
|
||||
},
|
||||
|
||||
// Postgres database
|
||||
@@ -35,30 +35,30 @@
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-postgres"],
|
||||
"environment": {
|
||||
"DATABASE_URL": "{env:DATABASE_URL}"
|
||||
"DATABASE_URL": "{env:DATABASE_URL}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// SQLite database
|
||||
"sqlite": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-sqlite"],
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Docker integration
|
||||
"docker": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-docker"],
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Web scraping
|
||||
"web-scraper": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "web-scraper-mcp"],
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Google Maps integration
|
||||
@@ -66,9 +66,9 @@
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-google-maps"],
|
||||
"environment": {
|
||||
"GOOGLE_MAPS_API_KEY": "{env:GOOGLE_MAPS_API_KEY}"
|
||||
"GOOGLE_MAPS_API_KEY": "{env:GOOGLE_MAPS_API_KEY}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Slack integration
|
||||
@@ -76,16 +76,16 @@
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-slack"],
|
||||
"environment": {
|
||||
"SLACK_BOT_TOKEN": "{env:SLACK_BOT_TOKEN}"
|
||||
"SLACK_BOT_TOKEN": "{env:SLACK_BOT_TOKEN}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Memory/knowledge base
|
||||
"memory": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-memory"],
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// AWS integration
|
||||
@@ -95,9 +95,9 @@
|
||||
"environment": {
|
||||
"AWS_ACCESS_KEY_ID": "{env:AWS_ACCESS_KEY_ID}",
|
||||
"AWS_SECRET_ACCESS_KEY": "{env:AWS_SECRET_ACCESS_KEY}",
|
||||
"AWS_REGION": "{env:AWS_REGION}"
|
||||
"AWS_REGION": "{env:AWS_REGION}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Linear integration
|
||||
@@ -105,9 +105,9 @@
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-linear"],
|
||||
"environment": {
|
||||
"LINEAR_API_KEY": "{env:LINEAR_API_KEY}"
|
||||
"LINEAR_API_KEY": "{env:LINEAR_API_KEY}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// Knowledge search via Context7
|
||||
@@ -115,53 +115,60 @@
|
||||
"type": "remote",
|
||||
"url": "https://mcp.context7.com/mcp",
|
||||
"headers": {
|
||||
"CONTEXT7_API_KEY": "{env:CONTEXT7_API_KEY}"
|
||||
"CONTEXT7_API_KEY": "{env:CONTEXT7_API_KEY}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": false,
|
||||
},
|
||||
|
||||
// GitHub code search via Grep
|
||||
"gh_grep": {
|
||||
"type": "remote",
|
||||
"url": "https://mcp.grep.app",
|
||||
"enabled": true
|
||||
"enabled": true,
|
||||
},
|
||||
|
||||
// WAF intelligence orchestrator
|
||||
"waf_intel": {
|
||||
"type": "local",
|
||||
"command": ["python3", "waf_intel_mcp.py"],
|
||||
"command": ["/bin/bash", "/Users/sovereign/work-core/.secret/mcp/waf_intelligence.sh"],
|
||||
"enabled": true,
|
||||
"timeout": 300000
|
||||
"timeout": 300000,
|
||||
},
|
||||
|
||||
// GitLab integration
|
||||
"gitlab": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-gitlab"],
|
||||
"environment": {
|
||||
"GITLAB_TOKEN": "{env:GITLAB_TOKEN}",
|
||||
"GITLAB_URL": "{env:GITLAB_URL:https://gitlab.com}"
|
||||
},
|
||||
"enabled": false
|
||||
"command": ["/opt/homebrew/bin/python3", "-u", "/Users/sovereign/work-core/.secret/gitlab_mcp_opencode_proxy.py"],
|
||||
"enabled": true,
|
||||
},
|
||||
|
||||
// Cloudflare API integration
|
||||
"cloudflare": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-cloudflare"],
|
||||
"command": ["/bin/bash", "/Users/sovereign/work-core/.secret/mcp_cloudflare_safe.sh"],
|
||||
"environment": {
|
||||
"CLOUDFLARE_API_TOKEN": "{env:CLOUDFLARE_API_TOKEN}",
|
||||
"CLOUDFLARE_ACCOUNT_ID": "{env:CLOUDFLARE_ACCOUNT_ID}"
|
||||
"CLOUDFLARE_ACCOUNT_ID": "{env:CLOUDFLARE_ACCOUNT_ID}",
|
||||
},
|
||||
"enabled": false
|
||||
"enabled": true,
|
||||
},
|
||||
|
||||
// Akash docs + SDL helpers (read-only; no wallet/key handling)
|
||||
"akash_docs": {
|
||||
"type": "local",
|
||||
"command": ["python3", "-m", "cloudflare.mcp.akash_docs"],
|
||||
"environment": {
|
||||
"PYTHONPATH": "/Users/sovereign/work-core"
|
||||
},
|
||||
"enabled": false,
|
||||
"timeout": 300000,
|
||||
},
|
||||
|
||||
// Test server (remove in production)
|
||||
"test_everything": {
|
||||
"type": "local",
|
||||
"command": ["npx", "-y", "@modelcontextprotocol/server-everything"],
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
"enabled": false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
1
requirements-dev.txt
Normal file
1
requirements-dev.txt
Normal file
@@ -0,0 +1 @@
|
||||
pytest>=8.0.0,<9
|
||||
308
scripts/deploy_infrastructure.sh
Normal file
308
scripts/deploy_infrastructure.sh
Normal file
@@ -0,0 +1,308 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Cloudflare Infrastructure Deployment Automation
|
||||
# Automated Terraform deployment with safety checks and rollback capabilities
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
TERRAFORM_DIR="terraform"
|
||||
BACKUP_DIR="terraform_backups"
|
||||
STATE_FILE="terraform.tfstate"
|
||||
PLAN_FILE="deployment_plan.tfplan"
|
||||
LOG_FILE="deployment_$(date +%Y%m%d_%H%M%S).log"
|
||||
|
||||
# Function to log messages
|
||||
log() {
|
||||
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to log success
|
||||
success() {
|
||||
echo -e "${GREEN}✅ $1${NC}" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to log warning
|
||||
warning() {
|
||||
echo -e "${YELLOW}⚠️ $1${NC}" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to log error
|
||||
error() {
|
||||
echo -e "${RED}❌ $1${NC}" | tee -a "$LOG_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Function to check prerequisites
|
||||
check_prerequisites() {
|
||||
log "Checking prerequisites..."
|
||||
|
||||
# Check if .env file exists
|
||||
if [[ ! -f "../.env" ]]; then
|
||||
error "Missing .env file. Run setup_credentials.sh first."
|
||||
fi
|
||||
|
||||
# Source environment variables
|
||||
source "../.env"
|
||||
|
||||
# Check required variables
|
||||
if [[ -z "$CLOUDFLARE_API_TOKEN" ]]; then
|
||||
error "CLOUDFLARE_API_TOKEN not set in .env"
|
||||
fi
|
||||
|
||||
if [[ -z "$CLOUDFLARE_ACCOUNT_ID" ]]; then
|
||||
error "CLOUDFLARE_ACCOUNT_ID not set in .env"
|
||||
fi
|
||||
|
||||
# Check Terraform installation
|
||||
if ! command -v terraform &> /dev/null; then
|
||||
error "Terraform not found. Please install Terraform first."
|
||||
fi
|
||||
|
||||
# Check Terraform version
|
||||
TF_VERSION=$(terraform version | head -n1 | awk '{print $2}' | sed 's/v//')
|
||||
log "Terraform version: $TF_VERSION"
|
||||
|
||||
success "Prerequisites check passed"
|
||||
}
|
||||
|
||||
# Function to backup current state
|
||||
backup_state() {
|
||||
log "Creating backup of current state..."
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Backup state file if it exists
|
||||
if [[ -f "$STATE_FILE" ]]; then
|
||||
BACKUP_NAME="${BACKUP_DIR}/state_backup_$(date +%Y%m%d_%H%M%S).tfstate"
|
||||
cp "$STATE_FILE" "$BACKUP_NAME"
|
||||
success "State backed up to: $BACKUP_NAME"
|
||||
else
|
||||
warning "No existing state file found"
|
||||
fi
|
||||
|
||||
# Backup terraform.tfvars
|
||||
if [[ -f "terraform.tfvars" ]]; then
|
||||
cp "terraform.tfvars" "${BACKUP_DIR}/terraform.tfvars.backup"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to prepare terraform.tfvars
|
||||
prepare_config() {
|
||||
log "Preparing Terraform configuration..."
|
||||
|
||||
# Update terraform.tfvars with actual credentials
|
||||
cat > terraform.tfvars << EOF
|
||||
cloudflare_api_token = "$CLOUDFLARE_API_TOKEN"
|
||||
cloudflare_account_id = "$CLOUDFLARE_ACCOUNT_ID"
|
||||
cloudflare_account_name = "" # Use account_id from .env
|
||||
EOF
|
||||
|
||||
# Add optional Zone ID if set
|
||||
if [[ -n "$CLOUDFLARE_ZONE_ID" ]]; then
|
||||
echo "cloudflare_zone_id = \"$CLOUDFLARE_ZONE_ID\"" >> terraform.tfvars
|
||||
fi
|
||||
|
||||
success "Configuration prepared"
|
||||
}
|
||||
|
||||
# Function to initialize Terraform
|
||||
init_terraform() {
|
||||
log "Initializing Terraform..."
|
||||
|
||||
if terraform init -upgrade; then
|
||||
success "Terraform initialized successfully"
|
||||
else
|
||||
error "Terraform initialization failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to validate Terraform configuration
|
||||
validate_config() {
|
||||
log "Validating Terraform configuration..."
|
||||
|
||||
if terraform validate; then
|
||||
success "Configuration validation passed"
|
||||
else
|
||||
error "Configuration validation failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to create deployment plan
|
||||
create_plan() {
|
||||
log "Creating deployment plan..."
|
||||
|
||||
if terraform plan -out="$PLAN_FILE" -detailed-exitcode; then
|
||||
case $? in
|
||||
0)
|
||||
success "No changes needed"
|
||||
return 0
|
||||
;;
|
||||
2)
|
||||
success "Plan created successfully"
|
||||
return 2
|
||||
;;
|
||||
*)
|
||||
error "Plan creation failed"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
error "Plan creation failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to show plan summary
|
||||
show_plan_summary() {
|
||||
log "Plan Summary:"
|
||||
terraform show -json "$PLAN_FILE" | jq -r '
|
||||
.resource_changes[] |
|
||||
select(.change.actions != ["no-op"]) |
|
||||
"\(.change.actions | join(",")) \(.type).\(.name)"
|
||||
' | sort | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Function to confirm deployment
|
||||
confirm_deployment() {
|
||||
echo
|
||||
echo "=================================================="
|
||||
echo "🚀 DEPLOYMENT CONFIRMATION"
|
||||
echo "=================================================="
|
||||
echo
|
||||
echo "The following changes will be applied:"
|
||||
show_plan_summary
|
||||
echo
|
||||
echo "Log file: $LOG_FILE"
|
||||
echo "Backup directory: $BACKUP_DIR"
|
||||
echo
|
||||
read -p "Do you want to proceed with deployment? (y/n): " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log "Deployment cancelled by user"
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to apply deployment
|
||||
apply_deployment() {
|
||||
log "Applying deployment..."
|
||||
|
||||
if terraform apply "$PLAN_FILE"; then
|
||||
success "Deployment applied successfully"
|
||||
else
|
||||
error "Deployment failed"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to verify deployment
|
||||
verify_deployment() {
|
||||
log "Verifying deployment..."
|
||||
|
||||
# Check if resources were created successfully
|
||||
OUTPUTS=$(terraform output -json)
|
||||
|
||||
if [[ -n "$OUTPUTS" ]]; then
|
||||
success "Deployment verification passed"
|
||||
echo "Outputs:"
|
||||
terraform output
|
||||
else
|
||||
warning "No outputs generated - manual verification required"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to cleanup temporary files
|
||||
cleanup() {
|
||||
log "Cleaning up temporary files..."
|
||||
|
||||
if [[ -f "$PLAN_FILE" ]]; then
|
||||
rm "$PLAN_FILE"
|
||||
success "Plan file removed"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to show deployment summary
|
||||
deployment_summary() {
|
||||
echo
|
||||
echo "=================================================="
|
||||
echo "🎉 DEPLOYMENT SUMMARY"
|
||||
echo "=================================================="
|
||||
echo
|
||||
echo "✅ Infrastructure deployed successfully"
|
||||
echo "📋 Log file: $LOG_FILE"
|
||||
echo "💾 Backups: $BACKUP_DIR"
|
||||
echo "🌐 Resources deployed:"
|
||||
terraform state list
|
||||
echo
|
||||
echo "Next steps:"
|
||||
echo "1. Check Cloudflare dashboard for deployed resources"
|
||||
echo "2. Test DNS resolution for your domains"
|
||||
echo "3. Verify WAF rules are active"
|
||||
echo "4. Test tunnel connectivity"
|
||||
echo
|
||||
}
|
||||
|
||||
# Function to handle rollback
|
||||
rollback() {
|
||||
error "Deployment failed - rolling back..."
|
||||
|
||||
# Check if we have a backup
|
||||
LATEST_BACKUP=$(ls -t "${BACKUP_DIR}/state_backup_*.tfstate" 2>/dev/null | head -n1)
|
||||
|
||||
if [[ -n "$LATEST_BACKUP" ]]; then
|
||||
log "Restoring from backup: $LATEST_BACKUP"
|
||||
cp "$LATEST_BACKUP" "$STATE_FILE"
|
||||
warning "State restored from backup. Manual verification required."
|
||||
else
|
||||
error "No backup available for rollback"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main deployment function
|
||||
main() {
|
||||
echo "🚀 Cloudflare Infrastructure Deployment"
|
||||
echo "=================================================="
|
||||
echo
|
||||
|
||||
# Change to Terraform directory
|
||||
cd "$TERRAFORM_DIR" || error "Terraform directory not found"
|
||||
|
||||
# Set trap for cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
# Execute deployment steps
|
||||
check_prerequisites
|
||||
backup_state
|
||||
prepare_config
|
||||
init_terraform
|
||||
validate_config
|
||||
|
||||
# Create plan and check if changes are needed
|
||||
if create_plan; then
|
||||
case $? in
|
||||
0)
|
||||
success "No changes needed - infrastructure is up to date"
|
||||
exit 0
|
||||
;;
|
||||
2)
|
||||
confirm_deployment
|
||||
apply_deployment
|
||||
verify_deployment
|
||||
deployment_summary
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
# Handle errors
|
||||
trap 'rollback' ERR
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
421
scripts/incident_response_playbooks.py
Normal file
421
scripts/incident_response_playbooks.py
Normal file
@@ -0,0 +1,421 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cloudflare Incident Response Playbooks
|
||||
Standardized procedures for common infrastructure incidents
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class IncidentSeverity(str, Enum):
|
||||
"""Incident severity levels"""
|
||||
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class IncidentType(str, Enum):
|
||||
"""Types of infrastructure incidents"""
|
||||
|
||||
DNS_OUTAGE = "dns_outage"
|
||||
WAF_BYPASS = "waf_bypass"
|
||||
TUNNEL_FAILURE = "tunnel_failure"
|
||||
SECURITY_BREACH = "security_breach"
|
||||
CONFIGURATION_ERROR = "configuration_error"
|
||||
PERFORMANCE_DEGRADATION = "performance_degradation"
|
||||
|
||||
|
||||
@dataclass
|
||||
class IncidentResponse:
|
||||
"""Incident response procedure"""
|
||||
|
||||
incident_type: IncidentType
|
||||
severity: IncidentSeverity
|
||||
immediate_actions: List[str]
|
||||
investigation_steps: List[str]
|
||||
recovery_procedures: List[str]
|
||||
prevention_measures: List[str]
|
||||
escalation_path: List[str]
|
||||
time_to_resolve: str
|
||||
|
||||
|
||||
class IncidentResponsePlaybook:
|
||||
"""Collection of incident response playbooks"""
|
||||
|
||||
def __init__(self):
|
||||
self.playbooks = self._initialize_playbooks()
|
||||
|
||||
def _initialize_playbooks(self) -> Dict[IncidentType, IncidentResponse]:
|
||||
"""Initialize all incident response playbooks"""
|
||||
return {
|
||||
IncidentType.DNS_OUTAGE: IncidentResponse(
|
||||
incident_type=IncidentType.DNS_OUTAGE,
|
||||
severity=IncidentSeverity.HIGH,
|
||||
immediate_actions=[
|
||||
"Verify DNS resolution using external tools (dig, nslookup)",
|
||||
"Check Cloudflare DNS dashboard for zone status",
|
||||
"Review recent DNS changes in version control",
|
||||
"Verify origin server connectivity",
|
||||
"Check Cloudflare status page for service issues",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Examine DNS record changes in Git history",
|
||||
"Check Terraform state for unexpected modifications",
|
||||
"Review Cloudflare audit logs for recent changes",
|
||||
"Verify DNS propagation using multiple geographic locations",
|
||||
"Check for DNSSEC configuration issues",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Rollback recent DNS changes using Terraform",
|
||||
"Manually restore critical DNS records if needed",
|
||||
"Update TTL values for faster propagation",
|
||||
"Contact Cloudflare support if service-related",
|
||||
"Implement traffic rerouting if necessary",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement DNS change approval workflows",
|
||||
"Use Terraform plan/apply with peer review",
|
||||
"Monitor DNS resolution from multiple locations",
|
||||
"Implement automated DNS health checks",
|
||||
"Maintain backup DNS configurations",
|
||||
],
|
||||
escalation_path=[
|
||||
"Primary DNS Administrator",
|
||||
"Infrastructure Team Lead",
|
||||
"Cloudflare Support",
|
||||
"Security Team",
|
||||
],
|
||||
time_to_resolve="1-4 hours",
|
||||
),
|
||||
IncidentType.WAF_BYPASS: IncidentResponse(
|
||||
incident_type=IncidentType.WAF_BYPASS,
|
||||
severity=IncidentSeverity.CRITICAL,
|
||||
immediate_actions=[
|
||||
"Immediately review WAF event logs for suspicious activity",
|
||||
"Check for recent WAF rule modifications",
|
||||
"Verify WAF rule package status and mode",
|
||||
"Temporarily block suspicious IP addresses",
|
||||
"Enable challenge mode for suspicious traffic patterns",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Analyze WAF rule changes in version control",
|
||||
"Review Cloudflare firewall event logs",
|
||||
"Check for anomalous traffic patterns",
|
||||
"Verify WAF rule effectiveness using test payloads",
|
||||
"Examine rate limiting and threat score thresholds",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Rollback WAF rule changes to known good state",
|
||||
"Implement emergency WAF rules to block attack patterns",
|
||||
"Update threat intelligence feeds",
|
||||
"Increase security level for affected zones",
|
||||
"Deploy additional security measures (Bot Fight Mode, etc.)",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement WAF change approval workflows",
|
||||
"Regular security testing of WAF rules",
|
||||
"Monitor WAF event logs for anomalies",
|
||||
"Implement automated WAF rule validation",
|
||||
"Regular security awareness training",
|
||||
],
|
||||
escalation_path=[
|
||||
"Security Incident Response Team",
|
||||
"WAF Administrator",
|
||||
"Infrastructure Security Lead",
|
||||
"CISO/Management",
|
||||
],
|
||||
time_to_resolve="2-6 hours",
|
||||
),
|
||||
IncidentType.TUNNEL_FAILURE: IncidentResponse(
|
||||
incident_type=IncidentType.TUNNEL_FAILURE,
|
||||
severity=IncidentSeverity.MEDIUM,
|
||||
immediate_actions=[
|
||||
"Check Cloudflare Tunnel status and connectivity",
|
||||
"Verify origin server availability and configuration",
|
||||
"Check tunnel connector logs for errors",
|
||||
"Restart tunnel connector service if needed",
|
||||
"Verify DNS records point to correct tunnel endpoints",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Review recent tunnel configuration changes",
|
||||
"Check network connectivity between connector and Cloudflare",
|
||||
"Examine tunnel connector resource usage",
|
||||
"Verify certificate validity and renewal status",
|
||||
"Check for firewall/network policy changes",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Restart tunnel connector with updated configuration",
|
||||
"Rollback recent tunnel configuration changes",
|
||||
"Recreate tunnel connector if necessary",
|
||||
"Update DNS records to alternative endpoints",
|
||||
"Implement traffic failover mechanisms",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement tunnel health monitoring",
|
||||
"Use redundant tunnel configurations",
|
||||
"Regular tunnel connector updates and maintenance",
|
||||
"Monitor certificate expiration dates",
|
||||
"Implement automated tunnel failover",
|
||||
],
|
||||
escalation_path=[
|
||||
"Network Administrator",
|
||||
"Infrastructure Team",
|
||||
"Cloudflare Support",
|
||||
"Security Team",
|
||||
],
|
||||
time_to_resolve="1-3 hours",
|
||||
),
|
||||
IncidentType.SECURITY_BREACH: IncidentResponse(
|
||||
incident_type=IncidentType.SECURITY_BREACH,
|
||||
severity=IncidentSeverity.CRITICAL,
|
||||
immediate_actions=[
|
||||
"Isolate affected systems and services immediately",
|
||||
"Preserve logs and evidence for forensic analysis",
|
||||
"Change all relevant credentials and API tokens",
|
||||
"Notify security incident response team",
|
||||
"Implement emergency security controls",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Conduct forensic analysis of compromised systems",
|
||||
"Review Cloudflare audit logs for unauthorized access",
|
||||
"Check for API token misuse or unauthorized changes",
|
||||
"Examine DNS/WAF/Tunnel configuration changes",
|
||||
"Coordinate with legal and compliance teams",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Rotate all Cloudflare API tokens and credentials",
|
||||
"Restore configurations from verified backups",
|
||||
"Implement enhanced security monitoring",
|
||||
"Conduct post-incident security assessment",
|
||||
"Update incident response procedures based on lessons learned",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement multi-factor authentication",
|
||||
"Regular security audits and penetration testing",
|
||||
"Monitor for suspicious API activity",
|
||||
"Implement least privilege access controls",
|
||||
"Regular security awareness training",
|
||||
],
|
||||
escalation_path=[
|
||||
"Security Incident Response Team",
|
||||
"CISO/Management",
|
||||
"Legal Department",
|
||||
"External Security Consultants",
|
||||
],
|
||||
time_to_resolve="4-24 hours",
|
||||
),
|
||||
IncidentType.CONFIGURATION_ERROR: IncidentResponse(
|
||||
incident_type=IncidentType.CONFIGURATION_ERROR,
|
||||
severity=IncidentSeverity.MEDIUM,
|
||||
immediate_actions=[
|
||||
"Identify the specific configuration error",
|
||||
"Assess impact on services and users",
|
||||
"Check version control for recent changes",
|
||||
"Verify Terraform plan output for unexpected changes",
|
||||
"Communicate status to stakeholders",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Review Git commit history for configuration changes",
|
||||
"Examine Terraform state differences",
|
||||
"Check Cloudflare configuration against documented standards",
|
||||
"Verify configuration consistency across environments",
|
||||
"Identify root cause of configuration error",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Rollback configuration using Terraform",
|
||||
"Apply corrected configuration changes",
|
||||
"Verify service restoration and functionality",
|
||||
"Update configuration documentation",
|
||||
"Implement configuration validation checks",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement configuration change approval workflows",
|
||||
"Use infrastructure as code with peer review",
|
||||
"Implement automated configuration validation",
|
||||
"Regular configuration audits",
|
||||
"Maintain configuration documentation",
|
||||
],
|
||||
escalation_path=[
|
||||
"Configuration Administrator",
|
||||
"Infrastructure Team Lead",
|
||||
"Quality Assurance Team",
|
||||
"Management",
|
||||
],
|
||||
time_to_resolve="1-4 hours",
|
||||
),
|
||||
IncidentType.PERFORMANCE_DEGRADATION: IncidentResponse(
|
||||
incident_type=IncidentType.PERFORMANCE_DEGRADATION,
|
||||
severity=IncidentSeverity.LOW,
|
||||
immediate_actions=[
|
||||
"Monitor performance metrics and identify bottlenecks",
|
||||
"Check Cloudflare analytics for traffic patterns",
|
||||
"Verify origin server performance and resource usage",
|
||||
"Review recent configuration changes",
|
||||
"Implement temporary performance optimizations",
|
||||
],
|
||||
investigation_steps=[
|
||||
"Analyze performance metrics over time",
|
||||
"Check for DDoS attacks or abnormal traffic patterns",
|
||||
"Review caching configuration and hit rates",
|
||||
"Examine origin server response times",
|
||||
"Identify specific performance bottlenecks",
|
||||
],
|
||||
recovery_procedures=[
|
||||
"Optimize caching configuration",
|
||||
"Adjust performance settings (Polish, Mirage, etc.)",
|
||||
"Implement rate limiting if under attack",
|
||||
"Scale origin server resources if needed",
|
||||
"Update CDN configuration for better performance",
|
||||
],
|
||||
prevention_measures=[
|
||||
"Implement performance monitoring and alerting",
|
||||
"Regular performance testing and optimization",
|
||||
"Capacity planning and resource forecasting",
|
||||
"Implement automated scaling mechanisms",
|
||||
"Regular performance reviews and optimizations",
|
||||
],
|
||||
escalation_path=[
|
||||
"Performance Monitoring Team",
|
||||
"Infrastructure Team",
|
||||
"Application Development Team",
|
||||
"Management",
|
||||
],
|
||||
time_to_resolve="2-8 hours",
|
||||
),
|
||||
}
|
||||
|
||||
def get_playbook(self, incident_type: IncidentType) -> Optional[IncidentResponse]:
|
||||
"""Get the playbook for a specific incident type"""
|
||||
return self.playbooks.get(incident_type)
|
||||
|
||||
def list_playbooks(self) -> List[IncidentType]:
|
||||
"""List all available playbooks"""
|
||||
return list(self.playbooks.keys())
|
||||
|
||||
def execute_playbook(
|
||||
self, incident_type: IncidentType, custom_context: Optional[Dict] = None
|
||||
) -> Dict:
|
||||
"""Execute a specific incident response playbook"""
|
||||
playbook = self.get_playbook(incident_type)
|
||||
|
||||
if not playbook:
|
||||
return {"error": f"No playbook found for incident type: {incident_type}"}
|
||||
|
||||
execution_log = {
|
||||
"incident_type": incident_type.value,
|
||||
"severity": playbook.severity.value,
|
||||
"start_time": datetime.now().isoformat(),
|
||||
"steps_completed": [],
|
||||
"custom_context": custom_context or {},
|
||||
}
|
||||
|
||||
# Simulate execution (in real implementation, this would trigger actual actions)
|
||||
execution_log["steps_completed"].extend(
|
||||
[
|
||||
f"Initiated {incident_type.value} response procedure",
|
||||
f"Severity level: {playbook.severity.value}",
|
||||
"Notified escalation path contacts",
|
||||
]
|
||||
)
|
||||
|
||||
execution_log["estimated_resolution_time"] = playbook.time_to_resolve
|
||||
execution_log["completion_status"] = "in_progress"
|
||||
|
||||
return execution_log
|
||||
|
||||
|
||||
def main():
|
||||
"""Command-line interface for incident response playbooks"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Cloudflare Incident Response Playbooks"
|
||||
)
|
||||
parser.add_argument(
|
||||
"action", choices=["list", "show", "execute"], help="Action to perform"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--type", choices=[t.value for t in IncidentType], help="Incident type"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
playbook_manager = IncidentResponsePlaybook()
|
||||
|
||||
if args.action == "list":
|
||||
print("📋 Available Incident Response Playbooks:")
|
||||
print("-" * 50)
|
||||
for incident_type in playbook_manager.list_playbooks():
|
||||
playbook = playbook_manager.get_playbook(incident_type)
|
||||
if not playbook:
|
||||
continue
|
||||
|
||||
print(f"🔸 {incident_type.value}")
|
||||
print(f" Severity: {playbook.severity.value}")
|
||||
print(f" Resolution Time: {playbook.time_to_resolve}")
|
||||
print()
|
||||
|
||||
elif args.action == "show":
|
||||
if not args.type:
|
||||
print("❌ Error: --type argument required")
|
||||
return
|
||||
|
||||
try:
|
||||
incident_type = IncidentType(args.type)
|
||||
except ValueError:
|
||||
print(f"❌ Error: Invalid incident type: {args.type}")
|
||||
return
|
||||
|
||||
playbook = playbook_manager.get_playbook(incident_type)
|
||||
if not playbook:
|
||||
print(f"❌ Error: No playbook found for {args.type}")
|
||||
return
|
||||
|
||||
print(f"🔍 Incident Response Playbook: {incident_type.value}")
|
||||
print("=" * 60)
|
||||
print(f"Severity: {playbook.severity.value}")
|
||||
print(f"Estimated Resolution: {playbook.time_to_resolve}")
|
||||
|
||||
print("\n🚨 Immediate Actions:")
|
||||
for i, action in enumerate(playbook.immediate_actions, 1):
|
||||
print(f" {i}. {action}")
|
||||
|
||||
print("\n🔍 Investigation Steps:")
|
||||
for i, step in enumerate(playbook.investigation_steps, 1):
|
||||
print(f" {i}. {step}")
|
||||
|
||||
print("\n🔄 Recovery Procedures:")
|
||||
for i, procedure in enumerate(playbook.recovery_procedures, 1):
|
||||
print(f" {i}. {procedure}")
|
||||
|
||||
print("\n🛡️ Prevention Measures:")
|
||||
for i, measure in enumerate(playbook.prevention_measures, 1):
|
||||
print(f" {i}. {measure}")
|
||||
|
||||
print("\n📞 Escalation Path:")
|
||||
for i, contact in enumerate(playbook.escalation_path, 1):
|
||||
print(f" {i}. {contact}")
|
||||
|
||||
elif args.action == "execute":
|
||||
if not args.type:
|
||||
print("❌ Error: --type argument required")
|
||||
return
|
||||
|
||||
try:
|
||||
incident_type = IncidentType(args.type)
|
||||
except ValueError:
|
||||
print(f"❌ Error: Invalid incident type: {args.type}")
|
||||
return
|
||||
|
||||
result = playbook_manager.execute_playbook(incident_type)
|
||||
print(f"🚀 Executing {incident_type.value} Incident Response")
|
||||
print(f"📊 Result: {result}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
260
scripts/monitoring_dashboard.py
Normal file
260
scripts/monitoring_dashboard.py
Normal file
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cloudflare Infrastructure Monitoring Dashboard
|
||||
Provides real-time monitoring of Cloudflare resources and services
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any
|
||||
|
||||
|
||||
class CloudflareMonitor:
|
||||
def __init__(self):
|
||||
self.base_url = "https://api.cloudflare.com/client/v4"
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {os.getenv('CLOUDFLARE_API_TOKEN')}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
self.account_id = os.getenv("CLOUDFLARE_ACCOUNT_ID")
|
||||
|
||||
if not self.account_id or not os.getenv("CLOUDFLARE_API_TOKEN"):
|
||||
raise ValueError("Missing Cloudflare credentials in environment")
|
||||
|
||||
def make_request(self, endpoint: str) -> Dict[str, Any]:
|
||||
"""Make API request with error handling"""
|
||||
url = f"{self.base_url}{endpoint}"
|
||||
try:
|
||||
response = requests.get(url, headers=self.headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except requests.RequestException as e:
|
||||
return {"success": False, "errors": [str(e)]}
|
||||
|
||||
def get_account_info(self) -> Dict[str, Any]:
|
||||
"""Get account information"""
|
||||
return self.make_request(f"/accounts/{self.account_id}")
|
||||
|
||||
def get_zones(self) -> List[Dict[str, Any]]:
|
||||
"""Get all zones"""
|
||||
result = self.make_request(f"/zones?account.id={self.account_id}&per_page=50")
|
||||
return result.get("result", []) if result.get("success") else []
|
||||
|
||||
def get_zone_analytics(self, zone_id: str) -> Dict[str, Any]:
|
||||
"""Get zone analytics for the last hour"""
|
||||
since = (datetime.now() - timedelta(hours=1)).isoformat()
|
||||
return self.make_request(f"/zones/{zone_id}/analytics/dashboard?since={since}")
|
||||
|
||||
def get_waf_rules(self, zone_id: str) -> List[Dict[str, Any]]:
|
||||
"""Get WAF rules for a zone"""
|
||||
result = self.make_request(f"/zones/{zone_id}/firewall/waf/packages")
|
||||
if result.get("success"):
|
||||
packages = result.get("result", [])
|
||||
rules = []
|
||||
for package in packages:
|
||||
rules_result = self.make_request(
|
||||
f"/zones/{zone_id}/firewall/waf/packages/{package['id']}/rules"
|
||||
)
|
||||
if rules_result.get("success"):
|
||||
rules.extend(rules_result.get("result", []))
|
||||
return rules
|
||||
return []
|
||||
|
||||
def get_tunnels(self) -> List[Dict[str, Any]]:
|
||||
"""Get Cloudflare Tunnels"""
|
||||
result = self.make_request(f"/accounts/{self.account_id}/cfd_tunnel")
|
||||
return result.get("result", []) if result.get("success") else []
|
||||
|
||||
def get_dns_records(self, zone_id: str) -> List[Dict[str, Any]]:
|
||||
"""Get DNS records for a zone"""
|
||||
result = self.make_request(f"/zones/{zone_id}/dns_records?per_page=100")
|
||||
return result.get("result", []) if result.get("success") else []
|
||||
|
||||
def get_health_status(self) -> Dict[str, Any]:
|
||||
"""Get overall health status"""
|
||||
status = "healthy"
|
||||
issues = []
|
||||
|
||||
# Check zones
|
||||
zones = self.get_zones()
|
||||
if not zones:
|
||||
issues.append("No zones found")
|
||||
status = "warning"
|
||||
|
||||
# Check account access
|
||||
account_info = self.get_account_info()
|
||||
if not account_info.get("success"):
|
||||
issues.append("Account access failed")
|
||||
status = "critical"
|
||||
|
||||
return {"status": status, "issues": issues}
|
||||
|
||||
|
||||
def format_table(data: List[Dict[str, Any]], headers: List[str]) -> str:
|
||||
"""Format data as a table"""
|
||||
if not data:
|
||||
return "No data available"
|
||||
|
||||
# Calculate column widths
|
||||
col_widths = [len(header) for header in headers]
|
||||
for row in data:
|
||||
for i, header in enumerate(headers):
|
||||
value = str(row.get(header, ""))
|
||||
col_widths[i] = max(col_widths[i], len(value))
|
||||
|
||||
# Create header row
|
||||
header_row = " | ".join(
|
||||
header.ljust(col_widths[i]) for i, header in enumerate(headers)
|
||||
)
|
||||
separator = "-" * len(header_row)
|
||||
|
||||
# Create data rows
|
||||
rows = [header_row, separator]
|
||||
for row in data:
|
||||
row_data = [
|
||||
str(row.get(header, "")).ljust(col_widths[i])
|
||||
for i, header in enumerate(headers)
|
||||
]
|
||||
rows.append(" | ".join(row_data))
|
||||
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def main():
|
||||
print("🌐 Cloudflare Infrastructure Monitoring Dashboard")
|
||||
print("=" * 60)
|
||||
print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print()
|
||||
|
||||
try:
|
||||
monitor = CloudflareMonitor()
|
||||
|
||||
# Health check
|
||||
print("🔍 Health Status")
|
||||
print("-" * 30)
|
||||
health = monitor.get_health_status()
|
||||
status_emoji = {"healthy": "✅", "warning": "⚠️", "critical": "❌"}
|
||||
print(
|
||||
f"Status: {status_emoji.get(health['status'], '❓')} {health['status'].upper()}"
|
||||
)
|
||||
if health["issues"]:
|
||||
for issue in health["issues"]:
|
||||
print(f" - {issue}")
|
||||
print()
|
||||
|
||||
# Account information
|
||||
print("🏢 Account Information")
|
||||
print("-" * 30)
|
||||
account_info = monitor.get_account_info()
|
||||
if account_info.get("success"):
|
||||
account = account_info["result"]
|
||||
print(f"Name: {account.get('name', 'N/A')}")
|
||||
print(f"Type: {account.get('type', 'N/A')}")
|
||||
print(f"Created: {account.get('created_on', 'N/A')}")
|
||||
else:
|
||||
print("Failed to retrieve account information")
|
||||
print()
|
||||
|
||||
# Zones overview
|
||||
print("🌐 Zones Overview")
|
||||
print("-" * 30)
|
||||
zones = monitor.get_zones()
|
||||
zone_data = []
|
||||
for zone in zones[:10]: # Limit to first 10 zones
|
||||
zone_data.append(
|
||||
{
|
||||
"Name": zone.get("name", "N/A"),
|
||||
"Status": zone.get("status", "N/A"),
|
||||
"Plan": zone.get("plan", {}).get("name", "N/A"),
|
||||
"Development": zone.get("development_mode", "N/A"),
|
||||
}
|
||||
)
|
||||
|
||||
print(format_table(zone_data, ["Name", "Status", "Plan", "Development"]))
|
||||
print(f"Total zones: {len(zones)}")
|
||||
print()
|
||||
|
||||
# DNS Records (for first zone)
|
||||
dns_records = []
|
||||
waf_rules = []
|
||||
|
||||
if zones:
|
||||
first_zone = zones[0]
|
||||
print("📋 DNS Records (First Zone)")
|
||||
print("-" * 30)
|
||||
dns_records = monitor.get_dns_records(first_zone["id"])
|
||||
dns_data = []
|
||||
for record in dns_records[:15]: # Limit to first 15 records
|
||||
dns_data.append(
|
||||
{
|
||||
"Type": record.get("type", "N/A"),
|
||||
"Name": record.get("name", "N/A"),
|
||||
"Content": record.get("content", "N/A")[:40] + "..."
|
||||
if len(record.get("content", "")) > 40
|
||||
else record.get("content", "N/A"),
|
||||
}
|
||||
)
|
||||
|
||||
print(format_table(dns_data, ["Type", "Name", "Content"]))
|
||||
print(f"Total DNS records: {len(dns_records)}")
|
||||
print()
|
||||
|
||||
# Tunnels
|
||||
print("🔗 Cloudflare Tunnels")
|
||||
print("-" * 30)
|
||||
tunnels = monitor.get_tunnels()
|
||||
tunnel_data = []
|
||||
for tunnel in tunnels:
|
||||
tunnel_data.append(
|
||||
{
|
||||
"Name": tunnel.get("name", "N/A"),
|
||||
"Status": tunnel.get("status", "N/A"),
|
||||
"Connections": len(tunnel.get("connections", [])),
|
||||
}
|
||||
)
|
||||
|
||||
print(format_table(tunnel_data, ["Name", "Status", "Connections"]))
|
||||
print(f"Total tunnels: {len(tunnels)}")
|
||||
print()
|
||||
|
||||
# WAF Rules (for first zone)
|
||||
if zones:
|
||||
first_zone = zones[0]
|
||||
print("🛡️ WAF Rules (First Zone)")
|
||||
print("-" * 30)
|
||||
waf_rules = monitor.get_waf_rules(first_zone["id"])
|
||||
waf_data = []
|
||||
for rule in waf_rules[:10]: # Limit to first 10 rules
|
||||
waf_data.append(
|
||||
{
|
||||
"ID": rule.get("id", "N/A"),
|
||||
"Description": rule.get("description", "N/A")[:50] + "..."
|
||||
if len(rule.get("description", "")) > 50
|
||||
else rule.get("description", "N/A"),
|
||||
"Mode": rule.get("mode", "N/A"),
|
||||
}
|
||||
)
|
||||
|
||||
print(format_table(waf_data, ["ID", "Description", "Mode"]))
|
||||
print(f"Total WAF rules: {len(waf_rules)}")
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("📊 Summary")
|
||||
print("-" * 30)
|
||||
print(f"Zones: {len(zones)}")
|
||||
print(f"Tunnels: {len(tunnels)}")
|
||||
if zones:
|
||||
print(f"DNS Records (first zone): {len(dns_records)}")
|
||||
print(f"WAF Rules (first zone): {len(waf_rules)}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print("Please ensure your Cloudflare credentials are properly configured.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
221
scripts/setup_credentials.py
Normal file
221
scripts/setup_credentials.py
Normal file
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cloudflare Credential Setup Wizard
|
||||
Interactive script to guide users through configuring Cloudflare API credentials
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def validate_api_token(token):
|
||||
"""Validate Cloudflare API token format"""
|
||||
# Cloudflare API tokens are typically 40+ characters
|
||||
return len(token.strip()) >= 40
|
||||
|
||||
|
||||
def validate_account_id(account_id):
|
||||
"""Validate Cloudflare Account ID format"""
|
||||
# Account IDs are typically 32-character hex strings
|
||||
return re.match(r"^[a-f0-9]{32}$", account_id.strip(), re.IGNORECASE) is not None
|
||||
|
||||
|
||||
def validate_zone_id(zone_id):
|
||||
"""Validate Cloudflare Zone ID format"""
|
||||
# Zone IDs are also 32-character hex strings
|
||||
return re.match(r"^[a-f0-9]{32}$", zone_id.strip(), re.IGNORECASE) is not None
|
||||
|
||||
|
||||
def get_input(prompt, validation_func=None, secret=False):
|
||||
"""Get validated user input"""
|
||||
while True:
|
||||
try:
|
||||
if secret:
|
||||
import getpass
|
||||
|
||||
value = getpass.getpass(prompt)
|
||||
else:
|
||||
value = input(prompt)
|
||||
|
||||
if validation_func:
|
||||
if validation_func(value):
|
||||
return value
|
||||
else:
|
||||
print("❌ Invalid format. Please try again.")
|
||||
else:
|
||||
return value
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nSetup cancelled.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def create_env_file(env_vars):
|
||||
"""Create or update .env file with credentials"""
|
||||
env_path = Path(".env")
|
||||
|
||||
# Read existing .env if it exists
|
||||
existing_vars = {}
|
||||
if env_path.exists():
|
||||
with open(env_path, "r") as f:
|
||||
for line in f:
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
key, value = line.strip().split("=", 1)
|
||||
existing_vars[key] = value
|
||||
|
||||
# Update with new values
|
||||
existing_vars.update(env_vars)
|
||||
|
||||
# Write back
|
||||
with open(env_path, "w") as f:
|
||||
f.write("# OpenCode Environment Variables\n")
|
||||
f.write("# Generated by setup_credentials.py\n")
|
||||
f.write("# IMPORTANT: Never commit this file to git\n\n")
|
||||
|
||||
# Write Cloudflare section
|
||||
f.write(
|
||||
"# ============================================================================\n"
|
||||
)
|
||||
f.write("# CLOUDFLARE API CONFIGURATION\n")
|
||||
f.write(
|
||||
"# ============================================================================\n"
|
||||
)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
f.write(f'{key}="{value}"\n')
|
||||
|
||||
f.write("\n")
|
||||
|
||||
# Preserve other sections if they exist
|
||||
sections = {
|
||||
"GITHUB": [k for k in existing_vars.keys() if k.startswith("GITHUB")],
|
||||
"GITLAB": [k for k in existing_vars.keys() if k.startswith("GITLAB")],
|
||||
"OTHER": [
|
||||
k
|
||||
for k in existing_vars.keys()
|
||||
if k not in env_vars and not k.startswith(("GITHUB", "GITLAB"))
|
||||
],
|
||||
}
|
||||
|
||||
for section_name, keys in sections.items():
|
||||
if keys:
|
||||
f.write(
|
||||
f"# ============================================================================\n"
|
||||
)
|
||||
f.write(f"# {section_name} CONFIGURATION\n")
|
||||
f.write(
|
||||
f"# ============================================================================\n"
|
||||
)
|
||||
for key in keys:
|
||||
f.write(f'{key}="{existing_vars[key]}"\n')
|
||||
f.write("\n")
|
||||
|
||||
return env_path
|
||||
|
||||
|
||||
def main():
|
||||
print("🚀 Cloudflare Credential Setup Wizard")
|
||||
print("=" * 50)
|
||||
print()
|
||||
|
||||
print("This wizard will help you configure your Cloudflare API credentials.")
|
||||
print("You'll need:")
|
||||
print("1. Cloudflare API Token (with appropriate permissions)")
|
||||
print("2. Cloudflare Account ID")
|
||||
print("3. Optional: Zone ID for specific domain management")
|
||||
print()
|
||||
|
||||
# Check if we're in the right directory
|
||||
current_dir = Path.cwd()
|
||||
if "cloudflare" not in str(current_dir):
|
||||
print("⚠️ Warning: This script should be run from the cloudflare directory")
|
||||
print(f" Current directory: {current_dir}")
|
||||
proceed = get_input("Continue anyway? (y/n): ")
|
||||
if proceed.lower() != "y":
|
||||
print(
|
||||
"Please navigate to the cloudflare directory and run this script again."
|
||||
)
|
||||
return
|
||||
|
||||
# Collect credentials
|
||||
print("\n🔐 Cloudflare API Configuration")
|
||||
print("-" * 30)
|
||||
|
||||
# API Token
|
||||
print("\n📋 Step 1: Cloudflare API Token")
|
||||
print("Get your token from: https://dash.cloudflare.com/profile/api-tokens")
|
||||
print("Required permissions: Zone:DNS:Edit, Zone:Page Rules:Edit, Account:Read")
|
||||
api_token = get_input(
|
||||
"API Token: ", validation_func=validate_api_token, secret=True
|
||||
)
|
||||
|
||||
# Account ID
|
||||
print("\n🏢 Step 2: Cloudflare Account ID")
|
||||
print("Find your Account ID in the Cloudflare dashboard sidebar")
|
||||
print("Format: 32-character hex string (e.g., 1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p)")
|
||||
account_id = get_input("Account ID: ", validation_func=validate_account_id)
|
||||
|
||||
# Zone ID (optional)
|
||||
print("\n🌐 Step 3: Zone ID (Optional)")
|
||||
print("If you want to manage a specific domain, provide its Zone ID")
|
||||
print("Leave blank to skip")
|
||||
zone_id = get_input(
|
||||
"Zone ID (optional): ",
|
||||
validation_func=lambda x: x.strip() == "" or validate_zone_id(x),
|
||||
)
|
||||
|
||||
# Prepare environment variables
|
||||
env_vars = {"CLOUDFLARE_API_TOKEN": api_token, "CLOUDFLARE_ACCOUNT_ID": account_id}
|
||||
|
||||
if zone_id.strip():
|
||||
env_vars["CLOUDFLARE_ZONE_ID"] = zone_id
|
||||
|
||||
# Create .env file
|
||||
print("\n💾 Saving credentials...")
|
||||
env_path = create_env_file(env_vars)
|
||||
|
||||
# Set file permissions
|
||||
env_path.chmod(0o600) # Only user read/write
|
||||
|
||||
print(f"✅ Credentials saved to: {env_path}")
|
||||
print("🔒 File permissions set to 600 (owner read/write only)")
|
||||
|
||||
# Test configuration (basic validation only - no external dependencies)
|
||||
print("\n🧪 Validating credentials...")
|
||||
|
||||
# Basic format validation
|
||||
if validate_api_token(api_token) and validate_account_id(account_id):
|
||||
print("✅ Credential formats are valid")
|
||||
print("⚠️ Note: Full API connectivity test requires 'requests' module")
|
||||
print(" Install with: pip install requests")
|
||||
else:
|
||||
print("❌ Credential validation failed")
|
||||
print(" Please check your inputs and try again")
|
||||
|
||||
# Final instructions
|
||||
print("\n🎉 Setup Complete!")
|
||||
print("=" * 50)
|
||||
print("\nNext steps:")
|
||||
print("1. Source the environment file:")
|
||||
print(" source .env")
|
||||
print("\n2. Test Terraform configuration:")
|
||||
print(" cd terraform && terraform init && terraform plan")
|
||||
print("\n3. Deploy infrastructure:")
|
||||
print(" terraform apply")
|
||||
print("\n4. Start MCP servers:")
|
||||
print(" Check MCP_GUIDE.md for server startup instructions")
|
||||
print("\n📚 Documentation:")
|
||||
print("- USAGE_GUIDE.md - Complete usage instructions")
|
||||
print("- DEPLOYMENT_GUIDE.md - Deployment procedures")
|
||||
print("- MCP_GUIDE.md - MCP server management")
|
||||
|
||||
# Security reminder
|
||||
print("\n🔐 Security Reminder:")
|
||||
print("- Never commit .env to version control")
|
||||
print("- Use .gitignore to exclude .env files")
|
||||
print("- Consider using environment-specific .env files (.env.production, etc.)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
190
scripts/setup_credentials.sh
Normal file
190
scripts/setup_credentials.sh
Normal file
@@ -0,0 +1,190 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Cloudflare Credential Setup Script
|
||||
# Interactive script to configure Cloudflare API credentials
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Cloudflare Credential Setup Wizard"
|
||||
echo "=================================================="
|
||||
echo
|
||||
|
||||
echo "This script will help you configure your Cloudflare API credentials."
|
||||
echo "You'll need:"
|
||||
echo "1. Cloudflare API Token (with appropriate permissions)"
|
||||
echo "2. Cloudflare Account ID"
|
||||
echo "3. Optional: Zone ID for specific domain management"
|
||||
echo
|
||||
|
||||
# Check if we're in the right directory
|
||||
if [[ ! "$PWD" =~ "cloudflare" ]]; then
|
||||
echo "⚠️ Warning: This script should be run from the cloudflare directory"
|
||||
echo " Current directory: $PWD"
|
||||
read -p "Continue anyway? (y/n): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Please navigate to the cloudflare directory and run this script again."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Function to validate API token format
|
||||
validate_api_token() {
|
||||
local token="$1"
|
||||
# Cloudflare API tokens are typically 40+ characters
|
||||
[[ ${#token} -ge 40 ]]
|
||||
}
|
||||
|
||||
# Function to validate Account ID format
|
||||
validate_account_id() {
|
||||
local account_id="$1"
|
||||
# Account IDs are 32-character hex strings
|
||||
[[ "$account_id" =~ ^[a-f0-9]{32}$ ]]
|
||||
}
|
||||
|
||||
# Function to validate Zone ID format
|
||||
validate_zone_id() {
|
||||
local zone_id="$1"
|
||||
# Zone IDs are 32-character hex strings
|
||||
[[ "$zone_id" =~ ^[a-f0-9]{32}$ ]]
|
||||
}
|
||||
|
||||
# Function to get validated input
|
||||
get_validated_input() {
|
||||
local prompt="$1"
|
||||
local validation_func="$2"
|
||||
local secret="$3"
|
||||
|
||||
while true; do
|
||||
if [[ "$secret" == "true" ]]; then
|
||||
read -s -p "$prompt" value
|
||||
echo
|
||||
else
|
||||
read -p "$prompt" value
|
||||
fi
|
||||
|
||||
if [[ -n "$validation_func" ]]; then
|
||||
if $validation_func "$value"; then
|
||||
echo "$value"
|
||||
return
|
||||
else
|
||||
echo "❌ Invalid format. Please try again."
|
||||
fi
|
||||
else
|
||||
echo "$value"
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Collect credentials
|
||||
echo "🔐 Cloudflare API Configuration"
|
||||
echo "------------------------------"
|
||||
echo
|
||||
|
||||
# API Token
|
||||
echo "📋 Step 1: Cloudflare API Token"
|
||||
echo "Get your token from: https://dash.cloudflare.com/profile/api-tokens"
|
||||
echo "Required permissions: Zone:DNS:Edit, Zone:Page Rules:Edit, Account:Read"
|
||||
API_TOKEN=$(get_validated_input "API Token: " validate_api_token true)
|
||||
|
||||
# Account ID
|
||||
echo
|
||||
echo "🏢 Step 2: Cloudflare Account ID"
|
||||
echo "Find your Account ID in the Cloudflare dashboard sidebar"
|
||||
echo "Format: 32-character hex string (e.g., 1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p)"
|
||||
ACCOUNT_ID=$(get_validated_input "Account ID: " validate_account_id false)
|
||||
|
||||
# Zone ID (optional)
|
||||
echo
|
||||
echo "🌐 Step 3: Zone ID (Optional)"
|
||||
echo "If you want to manage a specific domain, provide its Zone ID"
|
||||
echo "Leave blank to skip"
|
||||
ZONE_ID=$(get_validated_input "Zone ID (optional): " "[[ -z \"\$1\" ]] || validate_zone_id \"\$1\"" false)
|
||||
|
||||
# Create .env file
|
||||
echo
|
||||
echo "💾 Saving credentials..."
|
||||
|
||||
# Read existing .env if it exists
|
||||
ENV_CONTENT=""
|
||||
if [[ -f ".env" ]]; then
|
||||
# Preserve existing non-Cloudflare variables
|
||||
while IFS= read -r line; do
|
||||
if [[ ! "$line" =~ ^CLOUDFLARE_ ]] && [[ ! "$line" =~ ^#.*CLOUDFLARE ]]; then
|
||||
ENV_CONTENT="$ENV_CONTENT$line\n"
|
||||
fi
|
||||
done < ".env"
|
||||
fi
|
||||
|
||||
# Create new .env content
|
||||
cat > .env << EOF
|
||||
# OpenCode Environment Variables
|
||||
# Generated by setup_credentials.sh
|
||||
# IMPORTANT: Never commit this file to git
|
||||
|
||||
# ============================================================================
|
||||
# CLOUDFLARE API CONFIGURATION
|
||||
# ============================================================================
|
||||
CLOUDFLARE_API_TOKEN="$API_TOKEN"
|
||||
CLOUDFLARE_ACCOUNT_ID="$ACCOUNT_ID"
|
||||
EOF
|
||||
|
||||
# Add Zone ID if provided
|
||||
if [[ -n "$ZONE_ID" ]]; then
|
||||
echo "CLOUDFLARE_ZONE_ID=\"$ZONE_ID\"" >> .env
|
||||
fi
|
||||
|
||||
# Add preserved content
|
||||
if [[ -n "$ENV_CONTENT" ]]; then
|
||||
echo >> .env
|
||||
echo "$ENV_CONTENT" >> .env
|
||||
fi
|
||||
|
||||
# Set secure permissions
|
||||
chmod 600 .env
|
||||
|
||||
echo "✅ Credentials saved to: .env"
|
||||
echo "🔒 File permissions set to 600 (owner read/write only)"
|
||||
|
||||
# Basic validation
|
||||
echo
|
||||
echo "🧪 Validating credentials..."
|
||||
if validate_api_token "$API_TOKEN" && validate_account_id "$ACCOUNT_ID"; then
|
||||
echo "✅ Credential formats are valid"
|
||||
echo "⚠️ Note: Full API connectivity test requires curl or python requests"
|
||||
else
|
||||
echo "❌ Credential validation failed"
|
||||
echo " Please check your inputs and try again"
|
||||
fi
|
||||
|
||||
# Final instructions
|
||||
echo
|
||||
echo "🎉 Setup Complete!"
|
||||
echo "=================================================="
|
||||
echo
|
||||
echo "Next steps:"
|
||||
echo "1. Source the environment file:"
|
||||
echo " source .env"
|
||||
echo
|
||||
echo "2. Test Terraform configuration:"
|
||||
echo " cd terraform && terraform init && terraform plan"
|
||||
echo
|
||||
echo "3. Deploy infrastructure:"
|
||||
echo " terraform apply"
|
||||
echo
|
||||
echo "4. Start MCP servers:"
|
||||
echo " Check MCP_GUIDE.md for server startup instructions"
|
||||
echo
|
||||
echo "📚 Documentation:"
|
||||
echo "- USAGE_GUIDE.md - Complete usage instructions"
|
||||
echo "- DEPLOYMENT_GUIDE.md - Deployment procedures"
|
||||
echo "- MCP_GUIDE.md - MCP server management"
|
||||
echo
|
||||
echo "🔐 Security Reminder:"
|
||||
echo "- Never commit .env to version control"
|
||||
echo "- Use .gitignore to exclude .env files"
|
||||
echo "- Consider using environment-specific .env files (.env.production, etc.)"
|
||||
|
||||
# Make script executable
|
||||
chmod +x "$0"
|
||||
309
scripts/terraform_state_manager.py
Normal file
309
scripts/terraform_state_manager.py
Normal file
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Terraform State Backup and Recovery Manager
|
||||
Automated state management with versioning and rollback capabilities
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import shutil
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import argparse
|
||||
|
||||
|
||||
class TerraformStateManager:
|
||||
"""Manage Terraform state backups and recovery"""
|
||||
|
||||
def __init__(
|
||||
self, terraform_dir: str = "terraform", backup_dir: str = "terraform_backups"
|
||||
):
|
||||
self.terraform_dir = Path(terraform_dir)
|
||||
self.backup_dir = Path(backup_dir)
|
||||
self.state_file = self.terraform_dir / "terraform.tfstate"
|
||||
self.backup_dir.mkdir(exist_ok=True)
|
||||
|
||||
def create_backup(self, description: str = "", auto_backup: bool = True) -> str:
|
||||
"""Create a backup of the current Terraform state"""
|
||||
if not self.state_file.exists():
|
||||
return "No state file found to backup"
|
||||
|
||||
# Generate backup filename with timestamp
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_filename = f"state_backup_{timestamp}.tfstate"
|
||||
backup_path = self.backup_dir / backup_filename
|
||||
|
||||
# Copy state file
|
||||
shutil.copy2(self.state_file, backup_path)
|
||||
|
||||
# Create metadata file
|
||||
metadata = {
|
||||
"timestamp": timestamp,
|
||||
"description": description,
|
||||
"auto_backup": auto_backup,
|
||||
"file_size": os.path.getsize(backup_path),
|
||||
"file_hash": self._calculate_file_hash(backup_path),
|
||||
}
|
||||
|
||||
metadata_path = backup_path.with_suffix(".json")
|
||||
with open(metadata_path, "w") as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
return f"Backup created: {backup_filename}"
|
||||
|
||||
def list_backups(self) -> List[Dict]:
|
||||
"""List all available backups"""
|
||||
backups = []
|
||||
|
||||
for file in self.backup_dir.glob("state_backup_*.tfstate"):
|
||||
metadata_file = file.with_suffix(".json")
|
||||
|
||||
backup_info = {
|
||||
"filename": file.name,
|
||||
"path": str(file),
|
||||
"size": file.stat().st_size,
|
||||
"modified": datetime.fromtimestamp(file.stat().st_mtime),
|
||||
}
|
||||
|
||||
if metadata_file.exists():
|
||||
with open(metadata_file, "r") as f:
|
||||
backup_info.update(json.load(f))
|
||||
|
||||
backups.append(backup_info)
|
||||
|
||||
# Sort by modification time (newest first)
|
||||
backups.sort(key=lambda x: x["modified"], reverse=True)
|
||||
return backups
|
||||
|
||||
def restore_backup(self, backup_filename: str, dry_run: bool = False) -> str:
|
||||
"""Restore a specific backup"""
|
||||
backup_path = self.backup_dir / backup_filename
|
||||
|
||||
if not backup_path.exists():
|
||||
return f"Backup file not found: {backup_filename}"
|
||||
|
||||
# Create backup of current state before restore
|
||||
if self.state_file.exists() and not dry_run:
|
||||
self.create_backup("Pre-restore backup", auto_backup=True)
|
||||
|
||||
if dry_run:
|
||||
return f"Dry run: Would restore {backup_filename}"
|
||||
|
||||
# Perform restore
|
||||
shutil.copy2(backup_path, self.state_file)
|
||||
|
||||
return f"State restored from: {backup_filename}"
|
||||
|
||||
def cleanup_old_backups(
|
||||
self, keep_days: int = 30, keep_count: int = 10
|
||||
) -> List[str]:
|
||||
"""Clean up old backups based on age and count"""
|
||||
backups = self.list_backups()
|
||||
|
||||
if not backups:
|
||||
return ["No backups found to clean up"]
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=keep_days)
|
||||
backups_to_delete = []
|
||||
|
||||
# Delete backups older than keep_days
|
||||
for backup in backups:
|
||||
if backup["modified"] < cutoff_date:
|
||||
backups_to_delete.append(backup)
|
||||
|
||||
# If we have more than keep_count backups, delete the oldest ones
|
||||
if len(backups) > keep_count:
|
||||
# Keep the newest keep_count backups
|
||||
backups_to_keep = backups[:keep_count]
|
||||
backups_to_delete.extend([b for b in backups if b not in backups_to_keep])
|
||||
|
||||
# Remove duplicates
|
||||
backups_to_delete = list({b["filename"]: b for b in backups_to_delete}.values())
|
||||
|
||||
deleted_files = []
|
||||
for backup in backups_to_delete:
|
||||
try:
|
||||
# Delete state file
|
||||
state_file = Path(backup["path"])
|
||||
if state_file.exists():
|
||||
state_file.unlink()
|
||||
deleted_files.append(state_file.name)
|
||||
|
||||
# Delete metadata file
|
||||
metadata_file = state_file.with_suffix(".json")
|
||||
if metadata_file.exists():
|
||||
metadata_file.unlink()
|
||||
deleted_files.append(metadata_file.name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error deleting {backup['filename']}: {e}")
|
||||
|
||||
return deleted_files
|
||||
|
||||
def verify_backup_integrity(self, backup_filename: str) -> Dict[str, bool]:
|
||||
"""Verify the integrity of a backup"""
|
||||
backup_path = self.backup_dir / backup_filename
|
||||
metadata_path = backup_path.with_suffix(".json")
|
||||
|
||||
if not backup_path.exists():
|
||||
return {"exists": False, "metadata_exists": False, "integrity": False}
|
||||
|
||||
if not metadata_path.exists():
|
||||
return {"exists": True, "metadata_exists": False, "integrity": False}
|
||||
|
||||
# Check file size and hash
|
||||
with open(metadata_path, "r") as f:
|
||||
metadata = json.load(f)
|
||||
|
||||
current_size = backup_path.stat().st_size
|
||||
current_hash = self._calculate_file_hash(backup_path)
|
||||
|
||||
size_matches = current_size == metadata.get("file_size", 0)
|
||||
hash_matches = current_hash == metadata.get("file_hash", "")
|
||||
|
||||
return {
|
||||
"exists": True,
|
||||
"metadata_exists": True,
|
||||
"size_matches": size_matches,
|
||||
"hash_matches": hash_matches,
|
||||
"integrity": size_matches and hash_matches,
|
||||
}
|
||||
|
||||
def get_state_statistics(self) -> Dict:
|
||||
"""Get statistics about current state and backups"""
|
||||
backups = self.list_backups()
|
||||
|
||||
stats = {
|
||||
"current_state_exists": self.state_file.exists(),
|
||||
"current_state_size": self.state_file.stat().st_size
|
||||
if self.state_file.exists()
|
||||
else 0,
|
||||
"backup_count": len(backups),
|
||||
"oldest_backup": min([b["modified"] for b in backups]) if backups else None,
|
||||
"newest_backup": max([b["modified"] for b in backups]) if backups else None,
|
||||
"total_backup_size": sum(b["size"] for b in backups),
|
||||
"backups_with_issues": [],
|
||||
}
|
||||
|
||||
# Check backup integrity
|
||||
for backup in backups:
|
||||
integrity = self.verify_backup_integrity(backup["filename"])
|
||||
if not integrity["integrity"]:
|
||||
stats["backups_with_issues"].append(
|
||||
{"filename": backup["filename"], "integrity": integrity}
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
def _calculate_file_hash(self, file_path: Path) -> str:
|
||||
"""Calculate SHA256 hash of a file"""
|
||||
hasher = hashlib.sha256()
|
||||
with open(file_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
"""Command-line interface for Terraform state management"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Terraform State Backup and Recovery Manager"
|
||||
)
|
||||
parser.add_argument(
|
||||
"action",
|
||||
choices=["backup", "list", "restore", "cleanup", "stats", "verify"],
|
||||
help="Action to perform",
|
||||
)
|
||||
parser.add_argument("--filename", help="Backup filename for restore/verify")
|
||||
parser.add_argument("--description", help="Description for backup")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Dry run mode")
|
||||
parser.add_argument(
|
||||
"--keep-days", type=int, default=30, help="Days to keep backups"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-count", type=int, default=10, help="Number of backups to keep"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--terraform-dir", default="terraform", help="Terraform directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backup-dir", default="terraform_backups", help="Backup directory"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
manager = TerraformStateManager(args.terraform_dir, args.backup_dir)
|
||||
|
||||
if args.action == "backup":
|
||||
result = manager.create_backup(
|
||||
args.description or "Manual backup", auto_backup=False
|
||||
)
|
||||
print(f"✅ {result}")
|
||||
|
||||
elif args.action == "list":
|
||||
backups = manager.list_backups()
|
||||
print("📋 Available Backups:")
|
||||
print("-" * 80)
|
||||
for backup in backups:
|
||||
print(f"📁 {backup['filename']}")
|
||||
print(f" Size: {backup['size']:,} bytes")
|
||||
print(f" Modified: {backup['modified'].strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
if "description" in backup:
|
||||
print(f" Description: {backup['description']}")
|
||||
print()
|
||||
|
||||
elif args.action == "restore":
|
||||
if not args.filename:
|
||||
print("❌ Error: --filename argument required for restore")
|
||||
return
|
||||
|
||||
result = manager.restore_backup(args.filename, args.dry_run)
|
||||
print(f"🔁 {result}")
|
||||
|
||||
elif args.action == "cleanup":
|
||||
deleted = manager.cleanup_old_backups(args.keep_days, args.keep_count)
|
||||
if deleted:
|
||||
print("🗑️ Cleaned up backups:")
|
||||
for filename in deleted:
|
||||
print(f" - {filename}")
|
||||
else:
|
||||
print("✅ No backups needed cleanup")
|
||||
|
||||
elif args.action == "stats":
|
||||
stats = manager.get_state_statistics()
|
||||
print("📊 Terraform State Statistics")
|
||||
print("-" * 40)
|
||||
print(
|
||||
f"Current state exists: {'✅' if stats['current_state_exists'] else '❌'}"
|
||||
)
|
||||
print(f"Current state size: {stats['current_state_size']:,} bytes")
|
||||
print(f"Backup count: {stats['backup_count']}")
|
||||
if stats["oldest_backup"]:
|
||||
print(f"Oldest backup: {stats['oldest_backup'].strftime('%Y-%m-%d')}")
|
||||
print(f"Newest backup: {stats['newest_backup'].strftime('%Y-%m-%d')}")
|
||||
print(f"Total backup size: {stats['total_backup_size']:,} bytes")
|
||||
|
||||
if stats["backups_with_issues"]:
|
||||
print(f"\n⚠️ Backups with issues: {len(stats['backups_with_issues'])}")
|
||||
for issue in stats["backups_with_issues"]:
|
||||
print(f" - {issue['filename']}")
|
||||
|
||||
elif args.action == "verify":
|
||||
if not args.filename:
|
||||
print("❌ Error: --filename argument required for verify")
|
||||
return
|
||||
|
||||
integrity = manager.verify_backup_integrity(args.filename)
|
||||
print(f"🔍 Integrity check for {args.filename}")
|
||||
print(f" File exists: {'✅' if integrity['exists'] else '❌'}")
|
||||
print(f" Metadata exists: {'✅' if integrity['metadata_exists'] else '❌'}")
|
||||
if integrity["metadata_exists"]:
|
||||
print(f" Size matches: {'✅' if integrity['size_matches'] else '❌'}")
|
||||
print(f" Hash matches: {'✅' if integrity['hash_matches'] else '❌'}")
|
||||
print(f" Overall integrity: {'✅' if integrity['integrity'] else '❌'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
393
scripts/waf-and-plan-invariants.sh
Normal file
393
scripts/waf-and-plan-invariants.sh
Normal file
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# WAF + PLAN INVARIANTS CHECKER
|
||||
# ============================================================================
|
||||
# Enforces security+plan gating invariants for VaultMesh Cloudflare IaC.
|
||||
# Run from repo root: bash scripts/waf-and-plan-invariants.sh
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 = All invariants pass
|
||||
# 1 = One or more invariants violated
|
||||
#
|
||||
# Governed by: RED-BOOK.md
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo "============================================"
|
||||
echo " VaultMesh WAF + Plan Invariants Check"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
|
||||
FAILED=0
|
||||
|
||||
echo "── 0. Toolchain Versions ──"
|
||||
terraform version || true
|
||||
python3 --version || true
|
||||
python3 -m pip --version || true
|
||||
python3 -m pytest --version || true
|
||||
python3 -m mcp.waf_intelligence --version || true
|
||||
|
||||
echo ""
|
||||
|
||||
echo "── 1. WAF Intel Analyzer Regression ──"
|
||||
if python3 -m pytest -q tests/test_waf_intelligence_analyzer.py; then
|
||||
echo -e "${GREEN}✓${NC} 1.1 Analyzer regression test passed"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 1.1 Analyzer regression test failed"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── 2. WAF Intel CLI Contract ──"
|
||||
|
||||
TMP_DIR="${TMPDIR:-/tmp}"
|
||||
WAF_JSON_FILE="$(mktemp -p "$TMP_DIR" waf-intel.XXXXXX.json)"
|
||||
if python3 -m mcp.waf_intelligence --file terraform/waf.tf --format json --limit 5 >"$WAF_JSON_FILE"; then
|
||||
if python3 - "$WAF_JSON_FILE" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
insights = payload.get("insights")
|
||||
if not isinstance(insights, list):
|
||||
raise SystemExit("waf_intel: insights is not a list")
|
||||
|
||||
if insights:
|
||||
raise SystemExit(f"waf_intel: expected 0 insights, got {len(insights)}")
|
||||
|
||||
print("ok")
|
||||
PY
|
||||
then
|
||||
echo -e "${GREEN}✓${NC} 2.1 WAF Intel JSON parses and insights are empty"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 2.1 WAF Intel JSON contract violated"
|
||||
cat "$WAF_JSON_FILE"
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 2.1 WAF Intel CLI failed"
|
||||
FAILED=1
|
||||
fi
|
||||
rm -f "$WAF_JSON_FILE"
|
||||
|
||||
echo ""
|
||||
echo "── 3. Terraform Format + Validate + Plan Gates ──"
|
||||
|
||||
cd terraform
|
||||
|
||||
if terraform fmt -check -recursive >/dev/null 2>&1; then
|
||||
echo -e "${GREEN}✓${NC} 3.1 Terraform formatting OK"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.1 Terraform formatting required"
|
||||
echo " Run: cd terraform && terraform fmt -recursive"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
terraform init -backend=false -input=false >/dev/null 2>&1
|
||||
if terraform validate -no-color >/dev/null 2>&1; then
|
||||
echo -e "${GREEN}✓${NC} 3.2 Terraform validate OK"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.2 Terraform validate failed"
|
||||
terraform validate -no-color
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
PLAN_FREE_OUT="$(mktemp -p "$TMP_DIR" tf-plan-free.XXXXXX.out)"
|
||||
PLAN_PRO_OUT="$(mktemp -p "$TMP_DIR" tf-plan-pro.XXXXXX.out)"
|
||||
PLAN_FREE_JSON="$(mktemp -p "$TMP_DIR" tf-plan-free.XXXXXX.json)"
|
||||
PLAN_PRO_JSON="$(mktemp -p "$TMP_DIR" tf-plan-pro.XXXXXX.json)"
|
||||
rm -f "$PLAN_FREE_OUT" "$PLAN_PRO_OUT"
|
||||
|
||||
if terraform plan -no-color -input=false -lock=false -refresh=false -out="$PLAN_FREE_OUT" -var-file=assurance_free.tfvars >/dev/null; then
|
||||
if terraform show -json "$PLAN_FREE_OUT" >"$PLAN_FREE_JSON"; then
|
||||
if output="$(
|
||||
python3 - "$PLAN_FREE_JSON" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json parse error: {e}")
|
||||
raise SystemExit(2)
|
||||
|
||||
resource_changes = payload.get("resource_changes")
|
||||
planned_values = payload.get("planned_values")
|
||||
|
||||
if not isinstance(resource_changes, list) or not isinstance(planned_values, dict):
|
||||
print("invalid plan json: missing resource_changes[] and/or planned_values{}")
|
||||
raise SystemExit(2)
|
||||
|
||||
addresses = [
|
||||
rc.get("address", "")
|
||||
for rc in resource_changes
|
||||
if isinstance(rc, dict) and isinstance(rc.get("address"), str)
|
||||
]
|
||||
|
||||
managed_waf = sum(1 for a in addresses if a.startswith("cloudflare_ruleset.managed_waf["))
|
||||
bot_mgmt = sum(1 for a in addresses if a.startswith("cloudflare_bot_management.domains["))
|
||||
|
||||
if managed_waf != 0 or bot_mgmt != 0:
|
||||
print(f"expected managed_waf=0 bot_management=0, got managed_waf={managed_waf} bot_management={bot_mgmt}")
|
||||
for addr in sorted(
|
||||
a
|
||||
for a in addresses
|
||||
if a.startswith("cloudflare_ruleset.managed_waf[") or a.startswith("cloudflare_bot_management.domains[")
|
||||
):
|
||||
print(f"- {addr}")
|
||||
raise SystemExit(2)
|
||||
PY
|
||||
)"; then
|
||||
echo -e "${GREEN}✓${NC} 3.3 Free-plan gate OK (managed_waf=0 bot_management=0)"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.3 Free-plan gate violated"
|
||||
if [[ -n "${output:-}" ]]; then
|
||||
echo "$output" | sed 's/^/ /'
|
||||
fi
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.3 terraform show -json failed (free)"
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.3 Terraform plan failed (free)"
|
||||
terraform show -no-color "$PLAN_FREE_OUT" 2>/dev/null || true
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
if terraform plan -no-color -input=false -lock=false -refresh=false -out="$PLAN_PRO_OUT" -var-file=assurance_pro.tfvars >/dev/null; then
|
||||
if terraform show -json "$PLAN_PRO_OUT" >"$PLAN_PRO_JSON"; then
|
||||
if output="$(
|
||||
python3 - "$PLAN_PRO_JSON" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json parse error: {e}")
|
||||
raise SystemExit(2)
|
||||
|
||||
resource_changes = payload.get("resource_changes")
|
||||
planned_values = payload.get("planned_values")
|
||||
|
||||
if not isinstance(resource_changes, list) or not isinstance(planned_values, dict):
|
||||
print("invalid plan json: missing resource_changes[] and/or planned_values{}")
|
||||
raise SystemExit(2)
|
||||
|
||||
addresses = [
|
||||
rc.get("address", "")
|
||||
for rc in resource_changes
|
||||
if isinstance(rc, dict) and isinstance(rc.get("address"), str)
|
||||
]
|
||||
|
||||
managed_waf = sum(1 for a in addresses if a.startswith("cloudflare_ruleset.managed_waf["))
|
||||
bot_mgmt = sum(1 for a in addresses if a.startswith("cloudflare_bot_management.domains["))
|
||||
|
||||
if managed_waf != 1 or bot_mgmt != 1:
|
||||
print("expected managed_waf=1 bot_management=1")
|
||||
print(f"got managed_waf={managed_waf} bot_management={bot_mgmt}")
|
||||
print("observed:")
|
||||
for addr in sorted(
|
||||
a
|
||||
for a in addresses
|
||||
if a.startswith("cloudflare_ruleset.managed_waf[") or a.startswith("cloudflare_bot_management.domains[")
|
||||
):
|
||||
print(f"- {addr}")
|
||||
raise SystemExit(2)
|
||||
PY
|
||||
)"; then
|
||||
echo -e "${GREEN}✓${NC} 3.4 Paid-plan gate OK (managed_waf=1 bot_management=1)"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.4 Paid-plan gate violated"
|
||||
if [[ -n "${output:-}" ]]; then
|
||||
echo "$output" | sed 's/^/ /'
|
||||
fi
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.4 terraform show -json failed (pro)"
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 3.4 Terraform plan failed (pro)"
|
||||
terraform show -no-color "$PLAN_PRO_OUT" 2>/dev/null || true
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
PLAN_NEG_FREE_OUT="$(mktemp -p "$TMP_DIR" tf-plan-neg-free.XXXXXX.out)"
|
||||
PLAN_NEG_PRO_OUT="$(mktemp -p "$TMP_DIR" tf-plan-neg-pro.XXXXXX.out)"
|
||||
PLAN_NEG_FREE_JSON="$(mktemp -p "$TMP_DIR" tf-plan-neg-free.XXXXXX.json)"
|
||||
PLAN_NEG_PRO_JSON="$(mktemp -p "$TMP_DIR" tf-plan-neg-pro.XXXXXX.json)"
|
||||
rm -f "$PLAN_NEG_FREE_OUT" "$PLAN_NEG_PRO_OUT"
|
||||
|
||||
echo ""
|
||||
echo "── 4. Negative Controls (Prove the gate bites) ──"
|
||||
|
||||
if terraform plan -no-color -input=false -lock=false -refresh=false -out="$PLAN_NEG_FREE_OUT" -var-file=assurance_negative_free_should_fail.tfvars >/dev/null; then
|
||||
if terraform show -json "$PLAN_NEG_FREE_OUT" >"$PLAN_NEG_FREE_JSON"; then
|
||||
if output="$(
|
||||
python3 - "$PLAN_NEG_FREE_JSON" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json parse error: {e}")
|
||||
raise SystemExit(2)
|
||||
|
||||
resource_changes = payload.get("resource_changes")
|
||||
planned_values = payload.get("planned_values")
|
||||
|
||||
if not isinstance(resource_changes, list) or not isinstance(planned_values, dict):
|
||||
print("invalid plan json: missing resource_changes[] and/or planned_values{}")
|
||||
raise SystemExit(2)
|
||||
|
||||
addresses = [
|
||||
rc.get("address", "")
|
||||
for rc in resource_changes
|
||||
if isinstance(rc, dict) and isinstance(rc.get("address"), str)
|
||||
]
|
||||
|
||||
managed_waf = sum(1 for a in addresses if a.startswith("cloudflare_ruleset.managed_waf["))
|
||||
bot_mgmt = sum(1 for a in addresses if a.startswith("cloudflare_bot_management.domains["))
|
||||
|
||||
if managed_waf != 0 or bot_mgmt != 0:
|
||||
print(f"expected managed_waf=0 bot_management=0, got managed_waf={managed_waf} bot_management={bot_mgmt}")
|
||||
for addr in sorted(
|
||||
a
|
||||
for a in addresses
|
||||
if a.startswith("cloudflare_ruleset.managed_waf[") or a.startswith("cloudflare_bot_management.domains[")
|
||||
):
|
||||
print(f"- {addr}")
|
||||
raise SystemExit(2)
|
||||
|
||||
print("ok")
|
||||
PY
|
||||
)"; then
|
||||
echo -e "${RED}✗${NC} 4.1 Negative free-plan control unexpectedly passed"
|
||||
FAILED=1
|
||||
else
|
||||
if [[ "${output:-}" == *"expected managed_waf=0 bot_management=0"* ]]; then
|
||||
echo -e "${GREEN}✓${NC} 4.1 Negative free-plan control failed as expected"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.1 Negative free-plan control failed (unexpected error)"
|
||||
if [[ -n "${output:-}" ]]; then
|
||||
echo "$output" | sed 's/^/ /'
|
||||
fi
|
||||
FAILED=1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.1 terraform show -json failed (negative free)"
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.1 Terraform plan failed (negative free)"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
if terraform plan -no-color -input=false -lock=false -refresh=false -out="$PLAN_NEG_PRO_OUT" -var-file=assurance_negative_pro_should_fail.tfvars >/dev/null; then
|
||||
if terraform show -json "$PLAN_NEG_PRO_OUT" >"$PLAN_NEG_PRO_JSON"; then
|
||||
if output="$(
|
||||
python3 - "$PLAN_NEG_PRO_JSON" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json parse error: {e}")
|
||||
raise SystemExit(2)
|
||||
|
||||
resource_changes = payload.get("resource_changes")
|
||||
planned_values = payload.get("planned_values")
|
||||
|
||||
if not isinstance(resource_changes, list) or not isinstance(planned_values, dict):
|
||||
print("invalid plan json: missing resource_changes[] and/or planned_values{}")
|
||||
raise SystemExit(2)
|
||||
|
||||
addresses = [
|
||||
rc.get("address", "")
|
||||
for rc in resource_changes
|
||||
if isinstance(rc, dict) and isinstance(rc.get("address"), str)
|
||||
]
|
||||
|
||||
managed_waf = sum(1 for a in addresses if a.startswith("cloudflare_ruleset.managed_waf["))
|
||||
bot_mgmt = sum(1 for a in addresses if a.startswith("cloudflare_bot_management.domains["))
|
||||
|
||||
if managed_waf != 1 or bot_mgmt != 1:
|
||||
print("expected managed_waf=1 bot_management=1")
|
||||
print(f"got managed_waf={managed_waf} bot_management={bot_mgmt}")
|
||||
print("observed:")
|
||||
for addr in sorted(
|
||||
a
|
||||
for a in addresses
|
||||
if a.startswith("cloudflare_ruleset.managed_waf[") or a.startswith("cloudflare_bot_management.domains[")
|
||||
):
|
||||
print(f"- {addr}")
|
||||
raise SystemExit(2)
|
||||
|
||||
print("ok")
|
||||
PY
|
||||
)"; then
|
||||
echo -e "${RED}✗${NC} 4.2 Negative paid-plan control unexpectedly passed"
|
||||
FAILED=1
|
||||
else
|
||||
if [[ "${output:-}" == *"expected managed_waf=1 bot_management=1"* ]]; then
|
||||
echo -e "${GREEN}✓${NC} 4.2 Negative paid-plan control failed as expected"
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.2 Negative paid-plan control failed (unexpected error)"
|
||||
if [[ -n "${output:-}" ]]; then
|
||||
echo "$output" | sed 's/^/ /'
|
||||
fi
|
||||
FAILED=1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.2 terraform show -json failed (negative pro)"
|
||||
FAILED=1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}✗${NC} 4.2 Terraform plan failed (negative pro)"
|
||||
FAILED=1
|
||||
fi
|
||||
|
||||
rm -f "$PLAN_FREE_OUT" "$PLAN_PRO_OUT" "$PLAN_FREE_JSON" "$PLAN_PRO_JSON" "$PLAN_NEG_FREE_OUT" "$PLAN_NEG_PRO_OUT" "$PLAN_NEG_FREE_JSON" "$PLAN_NEG_PRO_JSON"
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " Summary"
|
||||
echo "============================================"
|
||||
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
echo -e "${RED}WAF + plan invariants violated. Fix before merging.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}All WAF + plan invariants pass. ✓${NC}"
|
||||
exit 0
|
||||
@@ -38,6 +38,8 @@ cloudflare_account_name = "your-account-name"
|
||||
tunnel_secret_vaultmesh = "base64-encoded-secret"
|
||||
tunnel_secret_offsec = "base64-encoded-secret"
|
||||
admin_emails = ["admin@vaultmesh.org"]
|
||||
enable_managed_waf = true
|
||||
enable_bot_management = false
|
||||
EOF
|
||||
|
||||
# Plan
|
||||
@@ -47,6 +49,31 @@ terraform plan
|
||||
terraform apply
|
||||
```
|
||||
|
||||
## Plan-Aware Security Features
|
||||
|
||||
- `enable_managed_waf` applies the managed WAF ruleset only when the zone `plan` is not `"free"`.
|
||||
- `enable_bot_management` applies bot management settings only when the zone `plan` is not `"free"`.
|
||||
|
||||
This lets `terraform apply` succeed on Free-plan zones (DNS, tunnels, Access, settings) while keeping the security posture ready for plan upgrades.
|
||||
|
||||
### WAF Truth Table
|
||||
|
||||
| Zone plan (`var.domains[*].plan`) | `enable_managed_waf` | `enable_bot_management` | Expected resources |
|
||||
| --- | --- | --- | --- |
|
||||
| `free` | any | any | `cloudflare_ruleset.security_rules` only |
|
||||
| not `free` | `false` | any | `cloudflare_ruleset.security_rules` only |
|
||||
| not `free` | `true` | `false` | `cloudflare_ruleset.security_rules`, `cloudflare_ruleset.managed_waf` |
|
||||
| not `free` | `true` | `true` | `cloudflare_ruleset.security_rules`, `cloudflare_ruleset.managed_waf`, `cloudflare_bot_management.domains` |
|
||||
|
||||
### Assurance Varfiles
|
||||
|
||||
For deterministic, token-format-safe gating checks (no apply), use:
|
||||
|
||||
```bash
|
||||
terraform plan -refresh=false -var-file=assurance_free.tfvars
|
||||
terraform plan -refresh=false -var-file=assurance_pro.tfvars
|
||||
```
|
||||
|
||||
## Generate Tunnel Secrets
|
||||
|
||||
```bash
|
||||
|
||||
35
terraform/assurance_free.tfvars
Normal file
35
terraform/assurance_free.tfvars
Normal file
@@ -0,0 +1,35 @@
|
||||
cloudflare_api_token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # Placeholder (format-valid)
|
||||
cloudflare_account_id = "00000000000000000000000000000000" # Placeholder (format-valid)
|
||||
cloudflare_account_name = ""
|
||||
|
||||
# Exercise empty-list safety
|
||||
trusted_admin_ips = []
|
||||
blocked_countries = []
|
||||
|
||||
# Even when flags are true, free-plan zones must gate these resources off
|
||||
enable_managed_waf = true
|
||||
enable_bot_management = true
|
||||
|
||||
# Keep the full set of expected zones so hard-coded references stay valid
|
||||
domains = {
|
||||
"offsec.global" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecglobal.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecagent.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecshield.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"vaultmesh.org" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
}
|
||||
34
terraform/assurance_negative_free_should_fail.tfvars
Normal file
34
terraform/assurance_negative_free_should_fail.tfvars
Normal file
@@ -0,0 +1,34 @@
|
||||
cloudflare_api_token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # Placeholder (format-valid)
|
||||
cloudflare_account_id = "00000000000000000000000000000000" # Placeholder (format-valid)
|
||||
cloudflare_account_name = ""
|
||||
|
||||
trusted_admin_ips = []
|
||||
blocked_countries = []
|
||||
|
||||
enable_managed_waf = true
|
||||
enable_bot_management = true
|
||||
|
||||
# Intentionally violates the "free plan must gate managed WAF + bot mgmt off".
|
||||
# Used by scripts/waf-and-plan-invariants.sh negative-control check.
|
||||
domains = {
|
||||
"offsec.global" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecglobal.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecagent.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecshield.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"vaultmesh.org" = {
|
||||
plan = "pro"
|
||||
jump_start = false
|
||||
}
|
||||
}
|
||||
34
terraform/assurance_negative_pro_should_fail.tfvars
Normal file
34
terraform/assurance_negative_pro_should_fail.tfvars
Normal file
@@ -0,0 +1,34 @@
|
||||
cloudflare_api_token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # Placeholder (format-valid)
|
||||
cloudflare_account_id = "00000000000000000000000000000000" # Placeholder (format-valid)
|
||||
cloudflare_account_name = ""
|
||||
|
||||
trusted_admin_ips = []
|
||||
blocked_countries = []
|
||||
|
||||
enable_managed_waf = true
|
||||
enable_bot_management = false
|
||||
|
||||
# Intentionally violates the "pro plan must create exactly 1 managed_waf + 1 bot_management" invariant.
|
||||
# Used by scripts/waf-and-plan-invariants.sh negative-control check.
|
||||
domains = {
|
||||
"offsec.global" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecglobal.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecagent.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecshield.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"vaultmesh.org" = {
|
||||
plan = "pro"
|
||||
jump_start = false
|
||||
}
|
||||
}
|
||||
34
terraform/assurance_pro.tfvars
Normal file
34
terraform/assurance_pro.tfvars
Normal file
@@ -0,0 +1,34 @@
|
||||
cloudflare_api_token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # Placeholder (format-valid)
|
||||
cloudflare_account_id = "00000000000000000000000000000000" # Placeholder (format-valid)
|
||||
cloudflare_account_name = ""
|
||||
|
||||
# Exercise empty-list safety
|
||||
trusted_admin_ips = []
|
||||
blocked_countries = []
|
||||
|
||||
enable_managed_waf = true
|
||||
enable_bot_management = true
|
||||
|
||||
# Mark at least one zone as non-free so plan includes managed WAF + bot mgmt resources.
|
||||
domains = {
|
||||
"offsec.global" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecglobal.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecagent.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"offsecshield.com" = {
|
||||
plan = "free"
|
||||
jump_start = false
|
||||
}
|
||||
"vaultmesh.org" = {
|
||||
plan = "pro"
|
||||
jump_start = false
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,7 @@ data "cloudflare_accounts" "main" {
|
||||
}
|
||||
|
||||
locals {
|
||||
# Use account ID from data source if available, otherwise use variable
|
||||
account_id = (
|
||||
var.cloudflare_account_name != "" && length(data.cloudflare_accounts.main) > 0 && length(data.cloudflare_accounts.main[0].accounts) > 0
|
||||
? data.cloudflare_accounts.main[0].accounts[0].id
|
||||
: var.cloudflare_account_id
|
||||
)
|
||||
# Use account ID from data source if available, otherwise fall back to variable.
|
||||
# `try()` avoids invalid index errors when the data source count is 0 or no accounts match.
|
||||
account_id = try(data.cloudflare_accounts.main[0].accounts[0].id, var.cloudflare_account_id)
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
cloudflare_api_token = "placeholder-token"
|
||||
cloudflare_account_id = "placeholder-account-id"
|
||||
cloudflare_account_name = "" # Leave empty to use hardcoded account_id
|
||||
cloudflare_api_token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # Placeholder (format-valid, not a real token)
|
||||
cloudflare_account_id = "00000000000000000000000000000000" # Placeholder (format-valid, not a real account ID)
|
||||
cloudflare_account_name = "" # Leave empty to use cloudflare_account_id
|
||||
|
||||
@@ -64,3 +64,15 @@ variable "blocked_countries" {
|
||||
type = list(string)
|
||||
default = ["CN", "RU", "KP", "IR"]
|
||||
}
|
||||
|
||||
variable "enable_managed_waf" {
|
||||
description = "Enable Cloudflare managed WAF rulesets (requires WAF entitlement; typically not available on Free plan)."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "enable_bot_management" {
|
||||
description = "Enable Cloudflare Bot Management settings (requires Bot Management entitlement)."
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ resource "cloudflare_ruleset" "security_rules" {
|
||||
# Rule 1: Block requests to /admin from non-trusted IPs
|
||||
rules {
|
||||
action = "block"
|
||||
expression = "(http.request.uri.path contains \"/admin\") and not (ip.src in {${join(" ", var.trusted_admin_ips)}})"
|
||||
expression = length(var.trusted_admin_ips) > 0 ? "(http.request.uri.path contains \"/admin\") and not (ip.src in {${join(" ", var.trusted_admin_ips)}})" : "false"
|
||||
description = "Block admin access from untrusted IPs"
|
||||
enabled = length(var.trusted_admin_ips) > 0
|
||||
}
|
||||
@@ -19,9 +19,9 @@ resource "cloudflare_ruleset" "security_rules" {
|
||||
# Rule 2: Challenge suspicious countries
|
||||
rules {
|
||||
action = "managed_challenge"
|
||||
expression = "(ip.src.country in {\"${join("\" \"", var.blocked_countries)}\"})"
|
||||
expression = length(var.blocked_countries) > 0 ? format("(ip.src.country in {%s})", join(" ", [for c in var.blocked_countries : format("\"%s\"", c)])) : "false"
|
||||
description = "Challenge traffic from high-risk countries"
|
||||
enabled = true
|
||||
enabled = length(var.blocked_countries) > 0
|
||||
}
|
||||
|
||||
# Rule 3: Block known bad user agents
|
||||
@@ -49,11 +49,14 @@ resource "cloudflare_ruleset" "security_rules" {
|
||||
|
||||
# Enable Cloudflare Managed WAF Ruleset
|
||||
resource "cloudflare_ruleset" "managed_waf" {
|
||||
for_each = cloudflare_zone.domains
|
||||
zone_id = each.value.id
|
||||
name = "Managed WAF"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_managed"
|
||||
for_each = {
|
||||
for domain, zone in cloudflare_zone.domains : domain => zone
|
||||
if var.enable_managed_waf && var.domains[domain].plan != "free"
|
||||
}
|
||||
zone_id = each.value.id
|
||||
name = "Managed WAF"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_managed"
|
||||
|
||||
# Cloudflare Managed Ruleset
|
||||
rules {
|
||||
@@ -80,7 +83,10 @@ resource "cloudflare_ruleset" "managed_waf" {
|
||||
|
||||
# Bot Management (if available on plan)
|
||||
resource "cloudflare_bot_management" "domains" {
|
||||
for_each = cloudflare_zone.domains
|
||||
for_each = {
|
||||
for domain, zone in cloudflare_zone.domains : domain => zone
|
||||
if var.enable_bot_management && var.domains[domain].plan != "free"
|
||||
}
|
||||
zone_id = each.value.id
|
||||
enable_js = true
|
||||
fight_mode = true
|
||||
|
||||
22
tests/test_mcp_cloudflare_safe_ingress.py
Normal file
22
tests/test_mcp_cloudflare_safe_ingress.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from mcp.cloudflare_safe.cloudflare_api import parse_cloudflared_config_ingress
|
||||
|
||||
|
||||
def test_parse_cloudflared_config_ingress_extracts_hostnames_and_services():
|
||||
sample = """\
|
||||
tunnel: 00000000-0000-0000-0000-000000000000
|
||||
credentials-file: /etc/cloudflared/0000.json
|
||||
|
||||
ingress:
|
||||
- hostname: "api.example.com"
|
||||
service: http://127.0.0.1:8080
|
||||
- hostname: app.example.com
|
||||
service: "http://127.0.0.1:3000"
|
||||
- service: http_status:404
|
||||
"""
|
||||
|
||||
rules = parse_cloudflared_config_ingress(sample)
|
||||
|
||||
assert rules == [
|
||||
{"hostname": "api.example.com", "service": "http://127.0.0.1:8080"},
|
||||
{"hostname": "app.example.com", "service": "http://127.0.0.1:3000"},
|
||||
]
|
||||
43
tests/test_waf_intelligence_analyzer.py
Normal file
43
tests/test_waf_intelligence_analyzer.py
Normal file
@@ -0,0 +1,43 @@
|
||||
from mcp.waf_intelligence.analyzer import WAFRuleAnalyzer
|
||||
|
||||
|
||||
def test_analyzer_detects_managed_waf_ruleset():
|
||||
analyzer = WAFRuleAnalyzer()
|
||||
|
||||
tf = """
|
||||
resource "cloudflare_ruleset" "managed_waf" {
|
||||
name = "Managed WAF"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_managed"
|
||||
|
||||
rules {
|
||||
action = "execute"
|
||||
action_parameters {
|
||||
id = "efb7b8c949ac4650a09736fc376e9aee"
|
||||
}
|
||||
expression = "true"
|
||||
description = "Execute Cloudflare Managed Ruleset"
|
||||
enabled = true
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
result = analyzer.analyze_terraform_text("snippet.tf", tf, min_severity="warning")
|
||||
assert result.violations == []
|
||||
|
||||
|
||||
def test_analyzer_warns_when_managed_waf_missing():
|
||||
analyzer = WAFRuleAnalyzer()
|
||||
|
||||
tf = """
|
||||
resource "cloudflare_ruleset" "security_rules" {
|
||||
name = "Security Rules"
|
||||
kind = "zone"
|
||||
phase = "http_request_firewall_custom"
|
||||
}
|
||||
"""
|
||||
|
||||
result = analyzer.analyze_terraform_text("snippet.tf", tf, min_severity="warning")
|
||||
assert [v.message for v in result.violations] == [
|
||||
"No managed WAF rules detected in this snippet."
|
||||
]
|
||||
60
validate_registry.sh
Executable file
60
validate_registry.sh
Executable file
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
# Local Registry Validation Script
|
||||
# Run this before commits to ensure registry integrity
|
||||
|
||||
echo "🔍 Local Registry Validation"
|
||||
echo "============================"
|
||||
|
||||
# Set Python path for MCP servers
|
||||
export PYTHONPATH="/Users/sovereign/work-core"
|
||||
|
||||
cd /Users/sovereign/work-core/cloudflare
|
||||
|
||||
# Generate fresh registry
|
||||
echo "📝 Generating fresh capability registry..."
|
||||
python3 generate_capability_registry_v2.py
|
||||
|
||||
# Check tool name parity
|
||||
echo "🔧 Checking tool name parity..."
|
||||
python3 ci_check_tool_names.py
|
||||
|
||||
# Check entrypoint sanity
|
||||
echo "🚀 Checking entrypoint sanity..."
|
||||
python3 ci_check_entrypoints.py
|
||||
|
||||
# Validate registry format
|
||||
echo "📊 Validating registry format..."
|
||||
python3 -c "
|
||||
import json
|
||||
with open('capability_registry_v2.json', 'r') as f:
|
||||
registry = json.load(f)
|
||||
|
||||
# Required sections
|
||||
required_sections = ['mcp_servers', 'terraform_resources', 'gitops_tools', 'security_framework', 'operational_tools']
|
||||
for section in required_sections:
|
||||
assert section in registry, f'Missing section: {section}'
|
||||
|
||||
# MCP server validation
|
||||
for server_name, server_info in registry['mcp_servers'].items():
|
||||
assert 'entrypoint' in server_info, f'Missing entrypoint for {server_name}'
|
||||
assert 'tools' in server_info, f'Missing tools for {server_name}'
|
||||
assert 'auth_env' in server_info, f'Missing auth_env for {server_name}'
|
||||
assert 'side_effects' in server_info, f'Missing side_effects for {server_name}'
|
||||
assert 'outputs' in server_info, f'Missing outputs for {server_name}'
|
||||
|
||||
print('✅ Registry format validation passed')
|
||||
"
|
||||
|
||||
# Check for changes from original
|
||||
echo "📈 Checking for registry changes..."
|
||||
if git diff --quiet capability_registry_v2.json; then
|
||||
echo "✅ Registry is stable - no changes detected"
|
||||
else
|
||||
echo "⚠️ Registry changed during validation"
|
||||
git diff capability_registry_v2.json
|
||||
echo "💡 Consider committing these changes"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🎉 Registry validation completed successfully!"
|
||||
echo "💡 Run this script before committing Cloudflare changes"
|
||||
109
waf_intel_mcp.py
109
waf_intel_mcp.py
@@ -1,110 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
from dataclasses import asdict
|
||||
from typing import Any, Dict, List
|
||||
"""
|
||||
WAF Intelligence MCP Server entrypoint.
|
||||
|
||||
from modelcontextprotocol.python import Server
|
||||
from mcp.waf_intelligence.orchestrator import WAFInsight, WAFIntelligence
|
||||
from layer0 import layer0_entry
|
||||
from layer0.shadow_classifier import ShadowEvalResult
|
||||
This wrapper intentionally avoids third-party MCP SDK dependencies and delegates to the
|
||||
in-repo stdio JSON-RPC implementation at `mcp.waf_intelligence.mcp_server`.
|
||||
"""
|
||||
|
||||
server = Server("waf_intel")
|
||||
|
||||
|
||||
def _insight_to_dict(insight: WAFInsight) -> Dict[str, Any]:
|
||||
"""Convert a WAFInsight dataclass into a plain dict."""
|
||||
return asdict(insight)
|
||||
|
||||
|
||||
@server.tool()
|
||||
async def analyze_waf(
|
||||
file: str | None = None,
|
||||
files: List[str] | None = None,
|
||||
limit: int = 3,
|
||||
severity_threshold: str = "warning",
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Analyze one or more Terraform WAF files and return curated insights.
|
||||
|
||||
Args:
|
||||
file: Single file path (e.g. "terraform/waf.tf").
|
||||
files: Optional list of file paths or glob patterns (e.g. ["terraform/waf*.tf"]).
|
||||
limit: Max number of high-priority insights to return.
|
||||
severity_threshold: Minimum severity to include ("info", "warning", "error").
|
||||
|
||||
Returns:
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"file": "...",
|
||||
"insights": [ ... ]
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
"""
|
||||
routing_action, shadow = layer0_entry(_shadow_repr(file, files, limit, severity_threshold))
|
||||
if routing_action != "HANDOFF_TO_LAYER1":
|
||||
_raise_layer0(routing_action, shadow)
|
||||
|
||||
paths: List[str] = []
|
||||
|
||||
if files:
|
||||
for pattern in files:
|
||||
for matched in glob.glob(pattern):
|
||||
paths.append(matched)
|
||||
|
||||
if file:
|
||||
paths.append(file)
|
||||
|
||||
seen = set()
|
||||
unique_paths: List[str] = []
|
||||
for p in paths:
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
unique_paths.append(p)
|
||||
|
||||
if not unique_paths:
|
||||
raise ValueError("Please provide 'file' or 'files' to analyze.")
|
||||
|
||||
intel = WAFIntelligence()
|
||||
results: List[Dict[str, Any]] = []
|
||||
|
||||
for path in unique_paths:
|
||||
insights: List[WAFInsight] = intel.analyze_and_recommend(
|
||||
path,
|
||||
limit=limit,
|
||||
min_severity=severity_threshold,
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"file": path,
|
||||
"insights": [_insight_to_dict(insight) for insight in insights],
|
||||
}
|
||||
)
|
||||
|
||||
return {"results": results}
|
||||
from cloudflare.mcp.waf_intelligence.mcp_server import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
server.run()
|
||||
|
||||
|
||||
def _shadow_repr(file: str | None, files: List[str] | None, limit: int, severity: str) -> str:
|
||||
try:
|
||||
return f"analyze_waf: file={file}, files={files}, limit={limit}, severity={severity}"
|
||||
except Exception:
|
||||
return "analyze_waf"
|
||||
|
||||
|
||||
def _raise_layer0(routing_action: str, shadow: ShadowEvalResult) -> None:
|
||||
if routing_action == "FAIL_CLOSED":
|
||||
raise ValueError("Layer 0: cannot comply with this request.")
|
||||
if routing_action == "HANDOFF_TO_GUARDRAILS":
|
||||
reason = shadow.reason or "governance_violation"
|
||||
raise ValueError(f"Layer 0: governance violation detected ({reason}).")
|
||||
if routing_action == "PROMPT_FOR_CLARIFICATION":
|
||||
raise ValueError("Layer 0: request is ambiguous. Please clarify and retry.")
|
||||
raise ValueError("Layer 0: unrecognized routing action; refusing request.")
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user