Upgrades and Migrations

# Snapshot all keyspaces
nodetool snapshot cadence
nodetool snapshot cadence_visibility

# Verify snapshots
nodetool listsnapshots

# Copy snapshots to remote storage
for host in $CASSANDRA_HOSTS; do
  ssh $host "tar czf /backup/cassandra-$(date +%Y%m%d).tar.gz \
    /var/lib/cassandra/data/*/snapshots/"
done

# Logical backup
mysqldump --single-transaction \
  --routines \
  --triggers \
  --databases cadence cadence_visibility \
  > cadence_backup_$(date +%Y%m%d).sql

# Or use Percona XtraBackup for large databases
xtrabackup --backup \
  --target-dir=/backup/cadence_$(date +%Y%m%d)

# Logical backup
pg_dump cadence > cadence_backup_$(date +%Y%m%d).sql

# Or physical backup
pg_basebackup -D /backup/cadence_$(date +%Y%m%d) -Ft -z -P

# Verify all services are healthy
curl http://frontend:9090/health
curl http://history:9091/health
curl http://matching:9092/health

# Check for stuck workflows
cadence admin workflow list --open --domain <domain>

# Verify no ongoing domain failovers
cadence admin domain list

# Backup current configuration
cp /etc/cadence/config.yaml /etc/cadence/config.yaml.backup

# Check for deprecated configuration options
grep -i deprecated config.yaml

schema/
  cassandra/
    cadence/
      versioned/
        v0.1/
        v0.2/
        ...
        v1.0/
    visibility/
      versioned/
        v0.1/
        ...
  mysql/
    v8/
      cadence/
        versioned/
          v0.1/
          ...
  postgres/
    v12/
      cadence/
        versioned/
          v0.1/
          ...

# Update cadence keyspace
cadence-cassandra-tool \
  --ep 127.0.0.1 \
  --keyspace cadence \
  update-schema \
  --version 1.0

# Update visibility keyspace
cadence-cassandra-tool \
  --ep 127.0.0.1 \
  --keyspace cadence_visibility \
  update-schema \
  --version 1.0

# Update cadence database
cadence-sql-tool \
  --ep 127.0.0.1:3306 \
  --db cadence \
  --plugin mysql \
  update-schema \
  --version 1.0

# Update visibility database
cadence-sql-tool \
  --ep 127.0.0.1:3306 \
  --db cadence_visibility \
  --plugin mysql \
  update-schema \
  --version 1.0

# Update cadence database
cadence-sql-tool \
  --ep 127.0.0.1:5432 \
  --db cadence \
  --plugin postgres \
  update-schema \
  --version 1.0

# Check current schema version
cadence-cassandra-tool \
  --ep 127.0.0.1 \
  --keyspace cadence \
  version

# Or for SQL
cadence-sql-tool \
  --ep 127.0.0.1:3306 \
  --db cadence \
  --plugin mysql \
  version

# Worker service is stateless and can be upgraded immediately
kubectl set image deployment/cadence-worker \
  cadence=ubercadence/server:1.0.0

# Or for systemd
systemctl stop cadence-worker
cp /opt/cadence/bin/cadence-server /opt/cadence/bin/cadence-server.old
cp /tmp/cadence-server-new /opt/cadence/bin/cadence-server
systemctl start cadence-worker

# Rolling upgrade with 25% max unavailable
kubectl rollout restart deployment/cadence-matching
kubectl rollout status deployment/cadence-matching

# Monitor for errors
kubectl logs -f deployment/cadence-matching --tail=100

# History service requires careful shard migration
# Use small batches to minimize shard transfer impact
kubectl set image deployment/cadence-history \
  cadence=ubercadence/server:1.0.0

# Monitor shard ownership changes
watch -n 5 'cadence admin shard list --print-full-shard'

# Frontend is stateless, safe to upgrade quickly
kubectl set image deployment/cadence-frontend \
  cadence=ubercadence/server:1.0.0

# Verify health
for i in {1..10}; do
  curl http://frontend:9090/health || echo "Failed"
  sleep 1
done

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cadence-history
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1      # Upgrade one pod at a time
      maxSurge: 1            # Allow one extra pod during upgrade
  template:
    spec:
      containers:
      - name: history
        image: ubercadence/server:1.0.0
        readinessProbe:
          httpGet:
            path: /health
            port: 9091
          initialDelaySeconds: 30
          periodSeconds: 10
        livenessProbe:
          httpGet:
            path: /health
            port: 9091
          initialDelaySeconds: 60
          periodSeconds: 30

# Old (deprecated)
clusterMetadata:
  masterClusterName: "primary"

# New
clusterGroupMetadata:
  primaryClusterName: "primary"

# Old (deprecated)
clusterMetadata:
  clusterInformation:
    cluster1: {...}

# New
clusterGroupMetadata:
  clusterGroup:
    cluster1: {...}

# Before 0.25.0
history.cacheSize: 1000

# After 0.25.0
history.historyCacheMaxSize: 1000

# Check all services
for svc in frontend history matching worker; do
  echo "Checking $svc..."
  curl -f http://$svc:909x/health || echo "$svc unhealthy!"
done

# Start a test workflow
cadence workflow start \
  --domain test-domain \
  --tasklist test-tasklist \
  --workflow_type TestWorkflow \
  --execution_timeout 60

# List workflows
cadence workflow list --domain test-domain

# Describe workflow
cadence workflow describe \
  --domain test-domain \
  --workflow_id <wf-id>

# Service restart count should increase by 1 per host
increase(cadence_restarts[10m])

# Error rate should remain low
rate(cadence_frontend_client_errors[5m]) < 0.01

# Latency should be normal
histogram_quantile(0.99, cadence_history_client_latency) < 1.0

# Check schema version matches target
cadence-cassandra-tool --ep 127.0.0.1 --keyspace cadence version

# Verify no persistence errors
grep -i "persistence error" /var/log/cadence/*.log

# Rollback to previous version
kubectl rollout undo deployment/cadence-history

# Or to specific revision
kubectl rollout undo deployment/cadence-history --to-revision=5

# Verify rollback
kubectl rollout status deployment/cadence-history

# Restore previous binary
for host in $CADENCE_HOSTS; do
  ssh $host "systemctl stop cadence-history && \
    cp /opt/cadence/bin/cadence-server.old \
       /opt/cadence/bin/cadence-server && \
    systemctl start cadence-history"
done

# Restore from snapshot
nodetool clearsnapshot cadence
nodetool refresh cadence <table>

# Or restore from backup
for host in $CASSANDRA_HOSTS; do
  ssh $host "systemctl stop cassandra && \
    rm -rf /var/lib/cassandra/data/cadence/* && \
    tar xzf /backup/cassandra-backup.tar.gz -C / && \
    systemctl start cassandra"
done

# Restore from backup
mysql cadence < cadence_backup.sql
mysql cadence_visibility < visibility_backup.sql

# Verify restoration
mysql -e "SELECT curr_version FROM schema_version" cadence

# Verify service versions
for pod in $(kubectl get pods -l app=cadence-history -o name); do
  kubectl describe $pod | grep Image:
done

# Check schema versions
cadence-cassandra-tool --ep 127.0.0.1 --keyspace cadence version

# Test workflow operations
cadence workflow start --domain test-domain ...

# Upgrade standby cluster 1
kubectl config use-context standby-1
kubectl set image deployment/cadence-history ...

# Verify replication
cadence admin domain describe --domain global-domain

# Repeat for other standby clusters
# Finally upgrade active cluster
kubectl config use-context active
kubectl set image deployment/cadence-history ...

# Increase timeout for large tables
cadence-cassandra-tool \
  --ep 127.0.0.1 \
  --keyspace cadence \
  --timeout 600 \
  update-schema --version 1.0

# Check for version skew
kubectl get pods -l app=cadence-history \
  -o jsonpath='{.items[*].spec.containers[0].image}'

# Verify all hosts have same version

# Check expected vs actual schema version
grep "schema version" /var/log/cadence/history.log

# Update schema to match
cadence-cassandra-tool update-schema --version X.Y

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: cadence-history
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: cadence-history
  progressDeadlineSeconds: 3600
  service:
    port: 7934
  analysis:
    interval: 1m
    threshold: 5
    maxWeight: 50
    stepWeight: 10
    metrics:
    - name: request-success-rate
      thresholdRange:
        min: 99
      interval: 1m