538 lines
15 KiB
Markdown
538 lines
15 KiB
Markdown
# VAULTMESH-MIGRATION-GUIDE.md
|
|
**Upgrading the Civilization Ledger**
|
|
|
|
> *A system that cannot evolve is a system that cannot survive.*
|
|
|
|
---
|
|
|
|
## 1. Version Compatibility Matrix
|
|
|
|
| From Version | To Version | Migration Type | Downtime |
|
|
|--------------|------------|----------------|----------|
|
|
| 0.1.x | 0.2.x | Schema migration | < 5 min |
|
|
| 0.2.x | 0.3.x | Schema migration | < 5 min |
|
|
| 0.3.x | 1.0.x | Major migration | < 30 min |
|
|
| 1.0.x | 1.1.x | Rolling update | None |
|
|
|
|
---
|
|
|
|
## 2. Pre-Migration Checklist
|
|
|
|
```bash
|
|
#!/bin/bash
|
|
# scripts/pre-migration-check.sh
|
|
|
|
set -e
|
|
|
|
echo "=== VaultMesh Pre-Migration Check ==="
|
|
|
|
# 1. Verify current version
|
|
CURRENT_VERSION=$(vm-cli version --short)
|
|
echo "Current version: $CURRENT_VERSION"
|
|
|
|
# 2. Check for pending anchors
|
|
PENDING=$(vm-guardian anchor-status --json | jq '.receipts_since_anchor')
|
|
if [ "$PENDING" -gt 0 ]; then
|
|
echo "WARNING: $PENDING receipts pending anchor"
|
|
echo "Running anchor before migration..."
|
|
vm-guardian anchor-now --wait
|
|
fi
|
|
|
|
# 3. Verify receipt integrity
|
|
echo "Verifying receipt integrity..."
|
|
vm-guardian verify-all --scroll all
|
|
if [ $? -ne 0 ]; then
|
|
echo "ERROR: Receipt integrity check failed"
|
|
exit 1
|
|
fi
|
|
|
|
# 4. Backup current state
|
|
echo "Creating backup..."
|
|
BACKUP_DIR="/backups/vaultmesh-$(date +%Y%m%d-%H%M%S)"
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
# Backup receipts
|
|
cp -r /data/receipts "$BACKUP_DIR/receipts"
|
|
|
|
# Backup database
|
|
pg_dump -h postgres -U vaultmesh vaultmesh > "$BACKUP_DIR/database.sql"
|
|
|
|
# Backup configuration
|
|
cp -r /config "$BACKUP_DIR/config"
|
|
|
|
# Backup Merkle roots
|
|
cp /data/receipts/ROOT.*.txt "$BACKUP_DIR/"
|
|
|
|
echo "Backup created: $BACKUP_DIR"
|
|
|
|
# 5. Verify backup
|
|
echo "Verifying backup..."
|
|
BACKUP_RECEIPT_COUNT=$(find "$BACKUP_DIR/receipts" -name "*.jsonl" -exec wc -l {} + | tail -1 | awk '{print $1}')
|
|
CURRENT_RECEIPT_COUNT=$(find /data/receipts -name "*.jsonl" -exec wc -l {} + | tail -1 | awk '{print $1}')
|
|
|
|
if [ "$BACKUP_RECEIPT_COUNT" -ne "$CURRENT_RECEIPT_COUNT" ]; then
|
|
echo "ERROR: Backup receipt count mismatch"
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== Pre-migration checks complete ==="
|
|
echo "Ready to migrate from $CURRENT_VERSION"
|
|
```
|
|
|
|
---
|
|
|
|
## 3. Migration Scripts
|
|
|
|
### 3.1 Schema Migration (0.2.x -> 0.3.x)
|
|
|
|
```python
|
|
# migrations/0002_to_0003.py
|
|
"""
|
|
Migration: 0.2.x -> 0.3.x
|
|
|
|
Changes:
|
|
- Add 'anchor_epoch' field to all receipts
|
|
- Add 'proof_path' field to all receipts
|
|
- Create new ROOT.*.txt files for new scrolls
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
import shutil
|
|
|
|
def migrate_receipts(receipts_dir: Path):
|
|
"""Add new fields to existing receipts."""
|
|
|
|
for jsonl_file in receipts_dir.glob("**/*.jsonl"):
|
|
print(f"Migrating: {jsonl_file}")
|
|
|
|
# Read all receipts
|
|
receipts = []
|
|
with open(jsonl_file) as f:
|
|
for line in f:
|
|
receipt = json.loads(line.strip())
|
|
|
|
# Add new fields if missing
|
|
if "anchor_epoch" not in receipt:
|
|
receipt["anchor_epoch"] = None
|
|
if "proof_path" not in receipt:
|
|
receipt["proof_path"] = None
|
|
|
|
receipts.append(receipt)
|
|
|
|
# Write back with new fields
|
|
backup_path = jsonl_file.with_suffix(".jsonl.bak")
|
|
shutil.copy(jsonl_file, backup_path)
|
|
|
|
with open(jsonl_file, "w") as f:
|
|
for receipt in receipts:
|
|
f.write(json.dumps(receipt) + "\n")
|
|
|
|
print(f" Migrated {len(receipts)} receipts")
|
|
|
|
def create_new_scrolls(receipts_dir: Path):
|
|
"""Create directories and root files for new scrolls."""
|
|
|
|
new_scrolls = [
|
|
"treasury",
|
|
"mesh",
|
|
"offsec",
|
|
"identity",
|
|
"observability",
|
|
"automation",
|
|
"psi",
|
|
"federation",
|
|
"governance",
|
|
]
|
|
|
|
for scroll in new_scrolls:
|
|
scroll_dir = receipts_dir / scroll
|
|
scroll_dir.mkdir(exist_ok=True)
|
|
|
|
# Create empty JSONL file
|
|
jsonl_file = scroll_dir / f"{scroll}_events.jsonl"
|
|
jsonl_file.touch()
|
|
|
|
# Create root file with empty root
|
|
root_file = receipts_dir / f"ROOT.{scroll}.txt"
|
|
root_file.write_text("blake3:empty")
|
|
|
|
print(f"Created scroll: {scroll}")
|
|
|
|
def update_database_schema():
|
|
"""Run database migrations."""
|
|
import subprocess
|
|
|
|
subprocess.run([
|
|
"sqlx", "migrate", "run",
|
|
"--source", "migrations/sql",
|
|
], check=True)
|
|
|
|
def main():
|
|
receipts_dir = Path("/data/receipts")
|
|
|
|
print("=== VaultMesh Migration: 0.2.x -> 0.3.x ===")
|
|
print(f"Timestamp: {datetime.utcnow().isoformat()}Z")
|
|
|
|
print("\n1. Migrating existing receipts...")
|
|
migrate_receipts(receipts_dir)
|
|
|
|
print("\n2. Creating new scroll directories...")
|
|
create_new_scrolls(receipts_dir)
|
|
|
|
print("\n3. Running database migrations...")
|
|
update_database_schema()
|
|
|
|
print("\n=== Migration complete ===")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
```
|
|
|
|
### 3.2 Major Migration (0.3.x -> 1.0.x)
|
|
|
|
```python
|
|
# migrations/0003_to_1000.py
|
|
"""
|
|
Migration: 0.3.x -> 1.0.x (Major)
|
|
|
|
Changes:
|
|
- Constitutional governance activation
|
|
- Receipt schema v2 (breaking)
|
|
- Merkle tree format change
|
|
- Guardian state restructure
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
import hashlib
|
|
import subprocess
|
|
import shutil
|
|
|
|
def backup_everything(backup_dir: Path):
|
|
"""Create comprehensive backup before major migration."""
|
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Full receipts backup with verification
|
|
receipts_backup = backup_dir / "receipts"
|
|
shutil.copytree("/data/receipts", receipts_backup)
|
|
|
|
# Compute checksums
|
|
checksums = {}
|
|
for f in receipts_backup.glob("**/*"):
|
|
if f.is_file():
|
|
checksums[str(f.relative_to(receipts_backup))] = hashlib.blake3(f.read_bytes()).hexdigest()
|
|
|
|
with open(backup_dir / "CHECKSUMS.json", "w") as f:
|
|
json.dump(checksums, f, indent=2)
|
|
|
|
# Database backup
|
|
subprocess.run([
|
|
"pg_dump", "-h", "postgres", "-U", "vaultmesh",
|
|
"-F", "c", # Custom format for parallel restore
|
|
"-f", str(backup_dir / "database.dump"),
|
|
"vaultmesh"
|
|
], check=True)
|
|
|
|
return backup_dir
|
|
|
|
def migrate_receipt_schema_v2(receipts_dir: Path):
|
|
"""Convert receipts to schema v2."""
|
|
|
|
for jsonl_file in receipts_dir.glob("**/*.jsonl"):
|
|
print(f"Converting to schema v2: {jsonl_file}")
|
|
|
|
receipts = []
|
|
with open(jsonl_file) as f:
|
|
for line in f:
|
|
old_receipt = json.loads(line.strip())
|
|
|
|
# Convert to v2 schema
|
|
new_receipt = {
|
|
"schema_version": "2.0.0",
|
|
"type": old_receipt.get("type"),
|
|
"timestamp": old_receipt.get("timestamp"),
|
|
"header": {
|
|
"root_hash": old_receipt.get("root_hash"),
|
|
"tags": old_receipt.get("tags", []),
|
|
"previous_hash": None, # Will be computed
|
|
},
|
|
"meta": {
|
|
"scroll": infer_scroll(jsonl_file),
|
|
"sequence": len(receipts),
|
|
"anchor_epoch": old_receipt.get("anchor_epoch"),
|
|
"proof_path": old_receipt.get("proof_path"),
|
|
},
|
|
"body": {
|
|
k: v for k, v in old_receipt.items()
|
|
if k not in ["type", "timestamp", "root_hash", "tags", "anchor_epoch", "proof_path"]
|
|
}
|
|
}
|
|
|
|
# Compute previous_hash chain
|
|
if receipts:
|
|
new_receipt["header"]["previous_hash"] = receipts[-1]["header"]["root_hash"]
|
|
|
|
# Recompute root_hash with new schema
|
|
new_receipt["header"]["root_hash"] = compute_receipt_hash_v2(new_receipt)
|
|
|
|
receipts.append(new_receipt)
|
|
|
|
# Write v2 receipts
|
|
with open(jsonl_file, "w") as f:
|
|
for receipt in receipts:
|
|
f.write(json.dumps(receipt) + "\n")
|
|
|
|
print(f" Converted {len(receipts)} receipts to v2")
|
|
|
|
def recompute_merkle_roots(receipts_dir: Path):
|
|
"""Recompute all Merkle roots with new format."""
|
|
|
|
scrolls = [
|
|
"drills", "compliance", "guardian", "treasury", "mesh",
|
|
"offsec", "identity", "observability", "automation",
|
|
"psi", "federation", "governance"
|
|
]
|
|
|
|
for scroll in scrolls:
|
|
jsonl_file = receipts_dir / scroll / f"{scroll}_events.jsonl"
|
|
root_file = receipts_dir / f"ROOT.{scroll}.txt"
|
|
|
|
if not jsonl_file.exists():
|
|
continue
|
|
|
|
# Read receipt hashes
|
|
hashes = []
|
|
with open(jsonl_file) as f:
|
|
for line in f:
|
|
receipt = json.loads(line.strip())
|
|
hashes.append(receipt["header"]["root_hash"])
|
|
|
|
# Compute new Merkle root
|
|
root = compute_merkle_root_v2(hashes)
|
|
root_file.write_text(root)
|
|
|
|
print(f"Recomputed root for {scroll}: {root[:30]}...")
|
|
|
|
def initialize_constitution():
|
|
"""Create initial constitutional documents."""
|
|
|
|
constitution = {
|
|
"version": "1.0.0",
|
|
"effective_at": datetime.utcnow().isoformat() + "Z",
|
|
"axioms": [], # From CONSTITUTIONAL-GOVERNANCE.md
|
|
"articles": [],
|
|
"engine_registry": [],
|
|
}
|
|
|
|
# Write constitution
|
|
const_path = Path("/data/governance/constitution.json")
|
|
const_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with open(const_path, "w") as f:
|
|
json.dump(constitution, f, indent=2)
|
|
|
|
# Create constitution receipt
|
|
receipt = {
|
|
"schema_version": "2.0.0",
|
|
"type": "gov_constitution_ratified",
|
|
"timestamp": datetime.utcnow().isoformat() + "Z",
|
|
"header": {
|
|
"root_hash": "", # Will be computed
|
|
"tags": ["governance", "constitution", "genesis"],
|
|
"previous_hash": None,
|
|
},
|
|
"meta": {
|
|
"scroll": "Governance",
|
|
"sequence": 0,
|
|
"anchor_epoch": None,
|
|
"proof_path": None,
|
|
},
|
|
"body": {
|
|
"constitution_version": "1.0.0",
|
|
"constitution_hash": hashlib.blake3(json.dumps(constitution).encode()).hexdigest(),
|
|
}
|
|
}
|
|
|
|
# Append to governance scroll
|
|
gov_jsonl = Path("/data/receipts/governance/governance_events.jsonl")
|
|
with open(gov_jsonl, "a") as f:
|
|
f.write(json.dumps(receipt) + "\n")
|
|
|
|
print("Constitutional governance initialized")
|
|
|
|
def main():
|
|
print("=== VaultMesh Major Migration: 0.3.x -> 1.0.x ===")
|
|
print(f"Timestamp: {datetime.utcnow().isoformat()}Z")
|
|
print("WARNING: This is a breaking migration!")
|
|
|
|
# Confirm
|
|
confirm = input("Type 'MIGRATE' to proceed: ")
|
|
if confirm != "MIGRATE":
|
|
print("Aborted")
|
|
return
|
|
|
|
backup_dir = Path(f"/backups/major-migration-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}")
|
|
receipts_dir = Path("/data/receipts")
|
|
|
|
print("\n1. Creating comprehensive backup...")
|
|
backup_everything(backup_dir)
|
|
|
|
print("\n2. Migrating receipt schema to v2...")
|
|
migrate_receipt_schema_v2(receipts_dir)
|
|
|
|
print("\n3. Recomputing Merkle roots...")
|
|
recompute_merkle_roots(receipts_dir)
|
|
|
|
print("\n4. Running database migrations...")
|
|
subprocess.run(["sqlx", "migrate", "run"], check=True)
|
|
|
|
print("\n5. Initializing constitutional governance...")
|
|
initialize_constitution()
|
|
|
|
print("\n6. Triggering anchor to seal migration...")
|
|
subprocess.run(["vm-guardian", "anchor-now", "--wait"], check=True)
|
|
|
|
print("\n=== Major migration complete ===")
|
|
print(f"Backup location: {backup_dir}")
|
|
print("Please verify system health before removing backup")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
```
|
|
|
|
---
|
|
|
|
## 4. Rollback Procedures
|
|
|
|
```bash
|
|
#!/bin/bash
|
|
# scripts/rollback.sh
|
|
|
|
set -e
|
|
|
|
BACKUP_DIR=$1
|
|
|
|
if [ -z "$BACKUP_DIR" ]; then
|
|
echo "Usage: rollback.sh <backup_directory>"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -d "$BACKUP_DIR" ]; then
|
|
echo "ERROR: Backup directory not found: $BACKUP_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== VaultMesh Rollback ==="
|
|
echo "Backup: $BACKUP_DIR"
|
|
|
|
# Verify backup integrity
|
|
echo "1. Verifying backup integrity..."
|
|
if [ -f "$BACKUP_DIR/CHECKSUMS.json" ]; then
|
|
python3 scripts/verify_checksums.py "$BACKUP_DIR"
|
|
fi
|
|
|
|
# Stop services
|
|
echo "2. Stopping services..."
|
|
kubectl scale deployment -n vaultmesh --replicas=0 \
|
|
vaultmesh-portal vaultmesh-guardian vaultmesh-oracle
|
|
|
|
# Restore database
|
|
echo "3. Restoring database..."
|
|
pg_restore -h postgres -U vaultmesh -d vaultmesh --clean "$BACKUP_DIR/database.dump"
|
|
|
|
# Restore receipts
|
|
echo "4. Restoring receipts..."
|
|
rm -rf /data/receipts/*
|
|
cp -r "$BACKUP_DIR/receipts"/* /data/receipts/
|
|
|
|
# Restore configuration
|
|
echo "5. Restoring configuration..."
|
|
cp -r "$BACKUP_DIR/config"/* /config/
|
|
|
|
# Restart services
|
|
echo "6. Restarting services..."
|
|
kubectl scale deployment -n vaultmesh --replicas=2 vaultmesh-portal
|
|
kubectl scale deployment -n vaultmesh --replicas=1 vaultmesh-guardian
|
|
kubectl scale deployment -n vaultmesh --replicas=2 vaultmesh-oracle
|
|
|
|
# Wait for health
|
|
echo "7. Waiting for services to become healthy..."
|
|
kubectl wait --for=condition=ready pod -l app.kubernetes.io/part-of=vaultmesh -n vaultmesh --timeout=300s
|
|
|
|
# Verify integrity
|
|
echo "8. Verifying receipt integrity..."
|
|
vm-guardian verify-all --scroll all
|
|
|
|
echo "=== Rollback complete ==="
|
|
```
|
|
|
|
---
|
|
|
|
## 5. Post-Migration Verification
|
|
|
|
```bash
|
|
#!/bin/bash
|
|
# scripts/post-migration-verify.sh
|
|
|
|
set -e
|
|
|
|
echo "=== VaultMesh Post-Migration Verification ==="
|
|
|
|
# 1. Version check
|
|
echo "1. Checking version..."
|
|
NEW_VERSION=$(vm-cli version --short)
|
|
echo " Version: $NEW_VERSION"
|
|
|
|
# 2. Service health
|
|
echo "2. Checking service health..."
|
|
vm-cli system health --json | jq '.services'
|
|
|
|
# 3. Receipt integrity
|
|
echo "3. Verifying receipt integrity..."
|
|
for scroll in drills compliance guardian treasury mesh offsec identity observability automation psi federation governance; do
|
|
COUNT=$(wc -l < "/data/receipts/$scroll/${scroll}_events.jsonl" 2>/dev/null || echo "0")
|
|
ROOT=$(cat "/data/receipts/ROOT.$scroll.txt" 2>/dev/null || echo "N/A")
|
|
echo " $scroll: $COUNT receipts, root: ${ROOT:0:20}..."
|
|
done
|
|
|
|
# 4. Merkle verification
|
|
echo "4. Verifying Merkle roots..."
|
|
vm-guardian verify-all --scroll all
|
|
|
|
# 5. Anchor status
|
|
echo "5. Checking anchor status..."
|
|
vm-guardian anchor-status
|
|
|
|
# 6. Constitution (if 1.0+)
|
|
if vm-gov constitution version &>/dev/null; then
|
|
echo "6. Checking constitution..."
|
|
vm-gov constitution version
|
|
fi
|
|
|
|
# 7. Test receipt emission
|
|
echo "7. Testing receipt emission..."
|
|
TEST_RECEIPT=$(vm-cli emit-test-receipt --scroll drills)
|
|
echo " Test receipt: $TEST_RECEIPT"
|
|
|
|
# 8. Test anchor
|
|
echo "8. Testing anchor cycle..."
|
|
vm-guardian anchor-now --wait
|
|
|
|
# 9. Verify test receipt was anchored
|
|
echo "9. Verifying test receipt anchored..."
|
|
PROOF=$(vm-guardian get-proof "$TEST_RECEIPT")
|
|
if [ -n "$PROOF" ]; then
|
|
echo " Test receipt successfully anchored"
|
|
else
|
|
echo " ERROR: Test receipt not anchored"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Post-migration verification complete ==="
|
|
echo "All checks passed. System is operational."
|
|
```
|