Initialize repository snapshot
This commit is contained in:
Vault Sovereign
537
docs/VAULTMESH-MIGRATION-GUIDE.md
Normal file
537
docs/VAULTMESH-MIGRATION-GUIDE.md
Normal file
@@ -0,0 +1,537 @@
|
||||
# VAULTMESH-MIGRATION-GUIDE.md
|
||||
**Upgrading the Civilization Ledger**
|
||||
|
||||
> *A system that cannot evolve is a system that cannot survive.*
|
||||
|
||||
---
|
||||
|
||||
## 1. Version Compatibility Matrix
|
||||
|
||||
| From Version | To Version | Migration Type | Downtime |
|
||||
|--------------|------------|----------------|----------|
|
||||
| 0.1.x | 0.2.x | Schema migration | < 5 min |
|
||||
| 0.2.x | 0.3.x | Schema migration | < 5 min |
|
||||
| 0.3.x | 1.0.x | Major migration | < 30 min |
|
||||
| 1.0.x | 1.1.x | Rolling update | None |
|
||||
|
||||
---
|
||||
|
||||
## 2. Pre-Migration Checklist
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# scripts/pre-migration-check.sh
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== VaultMesh Pre-Migration Check ==="
|
||||
|
||||
# 1. Verify current version
|
||||
CURRENT_VERSION=$(vm-cli version --short)
|
||||
echo "Current version: $CURRENT_VERSION"
|
||||
|
||||
# 2. Check for pending anchors
|
||||
PENDING=$(vm-guardian anchor-status --json | jq '.receipts_since_anchor')
|
||||
if [ "$PENDING" -gt 0 ]; then
|
||||
echo "WARNING: $PENDING receipts pending anchor"
|
||||
echo "Running anchor before migration..."
|
||||
vm-guardian anchor-now --wait
|
||||
fi
|
||||
|
||||
# 3. Verify receipt integrity
|
||||
echo "Verifying receipt integrity..."
|
||||
vm-guardian verify-all --scroll all
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Receipt integrity check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 4. Backup current state
|
||||
echo "Creating backup..."
|
||||
BACKUP_DIR="/backups/vaultmesh-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Backup receipts
|
||||
cp -r /data/receipts "$BACKUP_DIR/receipts"
|
||||
|
||||
# Backup database
|
||||
pg_dump -h postgres -U vaultmesh vaultmesh > "$BACKUP_DIR/database.sql"
|
||||
|
||||
# Backup configuration
|
||||
cp -r /config "$BACKUP_DIR/config"
|
||||
|
||||
# Backup Merkle roots
|
||||
cp /data/receipts/ROOT.*.txt "$BACKUP_DIR/"
|
||||
|
||||
echo "Backup created: $BACKUP_DIR"
|
||||
|
||||
# 5. Verify backup
|
||||
echo "Verifying backup..."
|
||||
BACKUP_RECEIPT_COUNT=$(find "$BACKUP_DIR/receipts" -name "*.jsonl" -exec wc -l {} + | tail -1 | awk '{print $1}')
|
||||
CURRENT_RECEIPT_COUNT=$(find /data/receipts -name "*.jsonl" -exec wc -l {} + | tail -1 | awk '{print $1}')
|
||||
|
||||
if [ "$BACKUP_RECEIPT_COUNT" -ne "$CURRENT_RECEIPT_COUNT" ]; then
|
||||
echo "ERROR: Backup receipt count mismatch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Pre-migration checks complete ==="
|
||||
echo "Ready to migrate from $CURRENT_VERSION"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Migration Scripts
|
||||
|
||||
### 3.1 Schema Migration (0.2.x -> 0.3.x)
|
||||
|
||||
```python
|
||||
# migrations/0002_to_0003.py
|
||||
"""
|
||||
Migration: 0.2.x -> 0.3.x
|
||||
|
||||
Changes:
|
||||
- Add 'anchor_epoch' field to all receipts
|
||||
- Add 'proof_path' field to all receipts
|
||||
- Create new ROOT.*.txt files for new scrolls
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
|
||||
def migrate_receipts(receipts_dir: Path):
|
||||
"""Add new fields to existing receipts."""
|
||||
|
||||
for jsonl_file in receipts_dir.glob("**/*.jsonl"):
|
||||
print(f"Migrating: {jsonl_file}")
|
||||
|
||||
# Read all receipts
|
||||
receipts = []
|
||||
with open(jsonl_file) as f:
|
||||
for line in f:
|
||||
receipt = json.loads(line.strip())
|
||||
|
||||
# Add new fields if missing
|
||||
if "anchor_epoch" not in receipt:
|
||||
receipt["anchor_epoch"] = None
|
||||
if "proof_path" not in receipt:
|
||||
receipt["proof_path"] = None
|
||||
|
||||
receipts.append(receipt)
|
||||
|
||||
# Write back with new fields
|
||||
backup_path = jsonl_file.with_suffix(".jsonl.bak")
|
||||
shutil.copy(jsonl_file, backup_path)
|
||||
|
||||
with open(jsonl_file, "w") as f:
|
||||
for receipt in receipts:
|
||||
f.write(json.dumps(receipt) + "\n")
|
||||
|
||||
print(f" Migrated {len(receipts)} receipts")
|
||||
|
||||
def create_new_scrolls(receipts_dir: Path):
|
||||
"""Create directories and root files for new scrolls."""
|
||||
|
||||
new_scrolls = [
|
||||
"treasury",
|
||||
"mesh",
|
||||
"offsec",
|
||||
"identity",
|
||||
"observability",
|
||||
"automation",
|
||||
"psi",
|
||||
"federation",
|
||||
"governance",
|
||||
]
|
||||
|
||||
for scroll in new_scrolls:
|
||||
scroll_dir = receipts_dir / scroll
|
||||
scroll_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Create empty JSONL file
|
||||
jsonl_file = scroll_dir / f"{scroll}_events.jsonl"
|
||||
jsonl_file.touch()
|
||||
|
||||
# Create root file with empty root
|
||||
root_file = receipts_dir / f"ROOT.{scroll}.txt"
|
||||
root_file.write_text("blake3:empty")
|
||||
|
||||
print(f"Created scroll: {scroll}")
|
||||
|
||||
def update_database_schema():
|
||||
"""Run database migrations."""
|
||||
import subprocess
|
||||
|
||||
subprocess.run([
|
||||
"sqlx", "migrate", "run",
|
||||
"--source", "migrations/sql",
|
||||
], check=True)
|
||||
|
||||
def main():
|
||||
receipts_dir = Path("/data/receipts")
|
||||
|
||||
print("=== VaultMesh Migration: 0.2.x -> 0.3.x ===")
|
||||
print(f"Timestamp: {datetime.utcnow().isoformat()}Z")
|
||||
|
||||
print("\n1. Migrating existing receipts...")
|
||||
migrate_receipts(receipts_dir)
|
||||
|
||||
print("\n2. Creating new scroll directories...")
|
||||
create_new_scrolls(receipts_dir)
|
||||
|
||||
print("\n3. Running database migrations...")
|
||||
update_database_schema()
|
||||
|
||||
print("\n=== Migration complete ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
### 3.2 Major Migration (0.3.x -> 1.0.x)
|
||||
|
||||
```python
|
||||
# migrations/0003_to_1000.py
|
||||
"""
|
||||
Migration: 0.3.x -> 1.0.x (Major)
|
||||
|
||||
Changes:
|
||||
- Constitutional governance activation
|
||||
- Receipt schema v2 (breaking)
|
||||
- Merkle tree format change
|
||||
- Guardian state restructure
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def backup_everything(backup_dir: Path):
|
||||
"""Create comprehensive backup before major migration."""
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Full receipts backup with verification
|
||||
receipts_backup = backup_dir / "receipts"
|
||||
shutil.copytree("/data/receipts", receipts_backup)
|
||||
|
||||
# Compute checksums
|
||||
checksums = {}
|
||||
for f in receipts_backup.glob("**/*"):
|
||||
if f.is_file():
|
||||
checksums[str(f.relative_to(receipts_backup))] = hashlib.blake3(f.read_bytes()).hexdigest()
|
||||
|
||||
with open(backup_dir / "CHECKSUMS.json", "w") as f:
|
||||
json.dump(checksums, f, indent=2)
|
||||
|
||||
# Database backup
|
||||
subprocess.run([
|
||||
"pg_dump", "-h", "postgres", "-U", "vaultmesh",
|
||||
"-F", "c", # Custom format for parallel restore
|
||||
"-f", str(backup_dir / "database.dump"),
|
||||
"vaultmesh"
|
||||
], check=True)
|
||||
|
||||
return backup_dir
|
||||
|
||||
def migrate_receipt_schema_v2(receipts_dir: Path):
|
||||
"""Convert receipts to schema v2."""
|
||||
|
||||
for jsonl_file in receipts_dir.glob("**/*.jsonl"):
|
||||
print(f"Converting to schema v2: {jsonl_file}")
|
||||
|
||||
receipts = []
|
||||
with open(jsonl_file) as f:
|
||||
for line in f:
|
||||
old_receipt = json.loads(line.strip())
|
||||
|
||||
# Convert to v2 schema
|
||||
new_receipt = {
|
||||
"schema_version": "2.0.0",
|
||||
"type": old_receipt.get("type"),
|
||||
"timestamp": old_receipt.get("timestamp"),
|
||||
"header": {
|
||||
"root_hash": old_receipt.get("root_hash"),
|
||||
"tags": old_receipt.get("tags", []),
|
||||
"previous_hash": None, # Will be computed
|
||||
},
|
||||
"meta": {
|
||||
"scroll": infer_scroll(jsonl_file),
|
||||
"sequence": len(receipts),
|
||||
"anchor_epoch": old_receipt.get("anchor_epoch"),
|
||||
"proof_path": old_receipt.get("proof_path"),
|
||||
},
|
||||
"body": {
|
||||
k: v for k, v in old_receipt.items()
|
||||
if k not in ["type", "timestamp", "root_hash", "tags", "anchor_epoch", "proof_path"]
|
||||
}
|
||||
}
|
||||
|
||||
# Compute previous_hash chain
|
||||
if receipts:
|
||||
new_receipt["header"]["previous_hash"] = receipts[-1]["header"]["root_hash"]
|
||||
|
||||
# Recompute root_hash with new schema
|
||||
new_receipt["header"]["root_hash"] = compute_receipt_hash_v2(new_receipt)
|
||||
|
||||
receipts.append(new_receipt)
|
||||
|
||||
# Write v2 receipts
|
||||
with open(jsonl_file, "w") as f:
|
||||
for receipt in receipts:
|
||||
f.write(json.dumps(receipt) + "\n")
|
||||
|
||||
print(f" Converted {len(receipts)} receipts to v2")
|
||||
|
||||
def recompute_merkle_roots(receipts_dir: Path):
|
||||
"""Recompute all Merkle roots with new format."""
|
||||
|
||||
scrolls = [
|
||||
"drills", "compliance", "guardian", "treasury", "mesh",
|
||||
"offsec", "identity", "observability", "automation",
|
||||
"psi", "federation", "governance"
|
||||
]
|
||||
|
||||
for scroll in scrolls:
|
||||
jsonl_file = receipts_dir / scroll / f"{scroll}_events.jsonl"
|
||||
root_file = receipts_dir / f"ROOT.{scroll}.txt"
|
||||
|
||||
if not jsonl_file.exists():
|
||||
continue
|
||||
|
||||
# Read receipt hashes
|
||||
hashes = []
|
||||
with open(jsonl_file) as f:
|
||||
for line in f:
|
||||
receipt = json.loads(line.strip())
|
||||
hashes.append(receipt["header"]["root_hash"])
|
||||
|
||||
# Compute new Merkle root
|
||||
root = compute_merkle_root_v2(hashes)
|
||||
root_file.write_text(root)
|
||||
|
||||
print(f"Recomputed root for {scroll}: {root[:30]}...")
|
||||
|
||||
def initialize_constitution():
|
||||
"""Create initial constitutional documents."""
|
||||
|
||||
constitution = {
|
||||
"version": "1.0.0",
|
||||
"effective_at": datetime.utcnow().isoformat() + "Z",
|
||||
"axioms": [], # From CONSTITUTIONAL-GOVERNANCE.md
|
||||
"articles": [],
|
||||
"engine_registry": [],
|
||||
}
|
||||
|
||||
# Write constitution
|
||||
const_path = Path("/data/governance/constitution.json")
|
||||
const_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(const_path, "w") as f:
|
||||
json.dump(constitution, f, indent=2)
|
||||
|
||||
# Create constitution receipt
|
||||
receipt = {
|
||||
"schema_version": "2.0.0",
|
||||
"type": "gov_constitution_ratified",
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"header": {
|
||||
"root_hash": "", # Will be computed
|
||||
"tags": ["governance", "constitution", "genesis"],
|
||||
"previous_hash": None,
|
||||
},
|
||||
"meta": {
|
||||
"scroll": "Governance",
|
||||
"sequence": 0,
|
||||
"anchor_epoch": None,
|
||||
"proof_path": None,
|
||||
},
|
||||
"body": {
|
||||
"constitution_version": "1.0.0",
|
||||
"constitution_hash": hashlib.blake3(json.dumps(constitution).encode()).hexdigest(),
|
||||
}
|
||||
}
|
||||
|
||||
# Append to governance scroll
|
||||
gov_jsonl = Path("/data/receipts/governance/governance_events.jsonl")
|
||||
with open(gov_jsonl, "a") as f:
|
||||
f.write(json.dumps(receipt) + "\n")
|
||||
|
||||
print("Constitutional governance initialized")
|
||||
|
||||
def main():
|
||||
print("=== VaultMesh Major Migration: 0.3.x -> 1.0.x ===")
|
||||
print(f"Timestamp: {datetime.utcnow().isoformat()}Z")
|
||||
print("WARNING: This is a breaking migration!")
|
||||
|
||||
# Confirm
|
||||
confirm = input("Type 'MIGRATE' to proceed: ")
|
||||
if confirm != "MIGRATE":
|
||||
print("Aborted")
|
||||
return
|
||||
|
||||
backup_dir = Path(f"/backups/major-migration-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}")
|
||||
receipts_dir = Path("/data/receipts")
|
||||
|
||||
print("\n1. Creating comprehensive backup...")
|
||||
backup_everything(backup_dir)
|
||||
|
||||
print("\n2. Migrating receipt schema to v2...")
|
||||
migrate_receipt_schema_v2(receipts_dir)
|
||||
|
||||
print("\n3. Recomputing Merkle roots...")
|
||||
recompute_merkle_roots(receipts_dir)
|
||||
|
||||
print("\n4. Running database migrations...")
|
||||
subprocess.run(["sqlx", "migrate", "run"], check=True)
|
||||
|
||||
print("\n5. Initializing constitutional governance...")
|
||||
initialize_constitution()
|
||||
|
||||
print("\n6. Triggering anchor to seal migration...")
|
||||
subprocess.run(["vm-guardian", "anchor-now", "--wait"], check=True)
|
||||
|
||||
print("\n=== Major migration complete ===")
|
||||
print(f"Backup location: {backup_dir}")
|
||||
print("Please verify system health before removing backup")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Rollback Procedures
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# scripts/rollback.sh
|
||||
|
||||
set -e
|
||||
|
||||
BACKUP_DIR=$1
|
||||
|
||||
if [ -z "$BACKUP_DIR" ]; then
|
||||
echo "Usage: rollback.sh <backup_directory>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$BACKUP_DIR" ]; then
|
||||
echo "ERROR: Backup directory not found: $BACKUP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== VaultMesh Rollback ==="
|
||||
echo "Backup: $BACKUP_DIR"
|
||||
|
||||
# Verify backup integrity
|
||||
echo "1. Verifying backup integrity..."
|
||||
if [ -f "$BACKUP_DIR/CHECKSUMS.json" ]; then
|
||||
python3 scripts/verify_checksums.py "$BACKUP_DIR"
|
||||
fi
|
||||
|
||||
# Stop services
|
||||
echo "2. Stopping services..."
|
||||
kubectl scale deployment -n vaultmesh --replicas=0 \
|
||||
vaultmesh-portal vaultmesh-guardian vaultmesh-oracle
|
||||
|
||||
# Restore database
|
||||
echo "3. Restoring database..."
|
||||
pg_restore -h postgres -U vaultmesh -d vaultmesh --clean "$BACKUP_DIR/database.dump"
|
||||
|
||||
# Restore receipts
|
||||
echo "4. Restoring receipts..."
|
||||
rm -rf /data/receipts/*
|
||||
cp -r "$BACKUP_DIR/receipts"/* /data/receipts/
|
||||
|
||||
# Restore configuration
|
||||
echo "5. Restoring configuration..."
|
||||
cp -r "$BACKUP_DIR/config"/* /config/
|
||||
|
||||
# Restart services
|
||||
echo "6. Restarting services..."
|
||||
kubectl scale deployment -n vaultmesh --replicas=2 vaultmesh-portal
|
||||
kubectl scale deployment -n vaultmesh --replicas=1 vaultmesh-guardian
|
||||
kubectl scale deployment -n vaultmesh --replicas=2 vaultmesh-oracle
|
||||
|
||||
# Wait for health
|
||||
echo "7. Waiting for services to become healthy..."
|
||||
kubectl wait --for=condition=ready pod -l app.kubernetes.io/part-of=vaultmesh -n vaultmesh --timeout=300s
|
||||
|
||||
# Verify integrity
|
||||
echo "8. Verifying receipt integrity..."
|
||||
vm-guardian verify-all --scroll all
|
||||
|
||||
echo "=== Rollback complete ==="
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Post-Migration Verification
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# scripts/post-migration-verify.sh
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== VaultMesh Post-Migration Verification ==="
|
||||
|
||||
# 1. Version check
|
||||
echo "1. Checking version..."
|
||||
NEW_VERSION=$(vm-cli version --short)
|
||||
echo " Version: $NEW_VERSION"
|
||||
|
||||
# 2. Service health
|
||||
echo "2. Checking service health..."
|
||||
vm-cli system health --json | jq '.services'
|
||||
|
||||
# 3. Receipt integrity
|
||||
echo "3. Verifying receipt integrity..."
|
||||
for scroll in drills compliance guardian treasury mesh offsec identity observability automation psi federation governance; do
|
||||
COUNT=$(wc -l < "/data/receipts/$scroll/${scroll}_events.jsonl" 2>/dev/null || echo "0")
|
||||
ROOT=$(cat "/data/receipts/ROOT.$scroll.txt" 2>/dev/null || echo "N/A")
|
||||
echo " $scroll: $COUNT receipts, root: ${ROOT:0:20}..."
|
||||
done
|
||||
|
||||
# 4. Merkle verification
|
||||
echo "4. Verifying Merkle roots..."
|
||||
vm-guardian verify-all --scroll all
|
||||
|
||||
# 5. Anchor status
|
||||
echo "5. Checking anchor status..."
|
||||
vm-guardian anchor-status
|
||||
|
||||
# 6. Constitution (if 1.0+)
|
||||
if vm-gov constitution version &>/dev/null; then
|
||||
echo "6. Checking constitution..."
|
||||
vm-gov constitution version
|
||||
fi
|
||||
|
||||
# 7. Test receipt emission
|
||||
echo "7. Testing receipt emission..."
|
||||
TEST_RECEIPT=$(vm-cli emit-test-receipt --scroll drills)
|
||||
echo " Test receipt: $TEST_RECEIPT"
|
||||
|
||||
# 8. Test anchor
|
||||
echo "8. Testing anchor cycle..."
|
||||
vm-guardian anchor-now --wait
|
||||
|
||||
# 9. Verify test receipt was anchored
|
||||
echo "9. Verifying test receipt anchored..."
|
||||
PROOF=$(vm-guardian get-proof "$TEST_RECEIPT")
|
||||
if [ -n "$PROOF" ]; then
|
||||
echo " Test receipt successfully anchored"
|
||||
else
|
||||
echo " ERROR: Test receipt not anchored"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Post-migration verification complete ==="
|
||||
echo "All checks passed. System is operational."
|
||||
```
|
||||
Reference in New Issue
Block a user