Operational Runbook

./scripts/dev_start.sh

# Load environment
export $(grep -vE '^\s*#' .env | xargs)

# Start server
uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-8000}

# In separate terminal: start ngrok
ngrok http ${PORT:-8000} --config ops/ngrok.yml

# Set production environment
export APP_ENV=prod

# Run with production-grade server
gunicorn app.main:app \
  --workers 4 \
  --worker-class uvicorn.workers.UvicornWorker \
  --bind 0.0.0.0:8000 \
  --access-logfile - \
  --error-logfile -

# Press Ctrl+C in the terminal running dev_start.sh
# The script handles cleanup of both uvicorn and ngrok processes

cleanup() {
  [[ -n "${NGROK_PID-}" ]] && kill "${NGROK_PID}" 2>/dev/null || true
}
trap cleanup EXIT INT TERM

# Kill uvicorn processes
pkill -f "uvicorn app.main:app"

# Kill ngrok processes
pkill -f ngrok

curl http://localhost:8000/health

{"status": "ok"}

wscat -c ws://localhost:8000/ws

curl https://your-ngrok-url.ngrok.io/health

# Edit .env
vim .env

# Update the relevant key
OPENAI_API_KEY=sk-new-key-here

# Restart required
./scripts/dev_start.sh

open http://localhost:8000/debug/live_calls/

open http://localhost:8000/debug/list_queue/

open http://localhost:8000/debug/list_calls/

curl http://localhost:8000/api/v1/queue | jq

[
  {
    "id": "call_session_123",
    "risk_level": "CRITICAL",
    "risk_score": 0.87,
    "category": "EMS",
    "tags": ["CARDIAC_ARREST", "UNCONSCIOUS"],
    "emotion": "HIGHLY_DISTRESSED",
    "summary": "Caller reports person collapsed and not breathing.",
    "status": "OPEN",
    "created_at": "2026-03-03T23:45:12Z",
    "from_masked": "•••5678",
    "to": "+15551234567"
  }
]

curl http://localhost:8000/api/v1/calls/{call_id} | jq

curl -X PATCH http://localhost:8000/api/v1/queue/{call_id}/status \
  -H "Content-Type: application/json" \
  -d '{
    "status": "IN_PROGRESS",
    "note": "Dispatching EMS unit 42"
  }'

curl http://localhost:8000/api/v1/live_queue | jq

# View recordings
ls -lh data/recordings/

# Location referenced in LIVE_SIGNALS dict
# wav_path: data/recordings/{call_id}.wav

python scripts/replay_call.py data/recordings/call_session_123.wav

python scripts/export_packets.py --output calls.json --limit 100

if APP_ENV == AppEnv.DEV:
    CALL_STORE: CallStore = InMemoryCallStore(max_recent=200)
else:
    # For now, still in-memory; later drop in Postgres-backed class here.
    CALL_STORE: CallStore = InMemoryCallStore(max_recent=200)

# 1. Check server is running
curl http://localhost:8000/health

# 2. Check ngrok tunnel
curl https://your-ngrok-url.ngrok.io/health

# 3. Verify webhook configuration
cat ops/urls.txt

# 4. Check Telnyx portal webhook settings

# Check logs for streaming_start errors
grep "streaming_start" logs.txt

# Verify WS_PUBLIC_URL is set
echo $WS_PUBLIC_URL

# Check for Deepgram errors
grep "deepgram error" logs.txt

# Test API key
curl -X POST https://api.deepgram.com/v1/listen \
  -H "Authorization: Token $DEEPGRAM_API_KEY" \
  -H "Content-Type: audio/wav" \
  --data-binary @test.wav

# Review audio analysis agent
cat app/agents/audio_track.py

# Check distress calculation
grep "distress_score" logs.txt | tail -20

# Check if OpenAI key is set
echo $OPENAI_API_KEY

# Look for summary errors
grep "summary" logs.txt | grep -i error

# Monitor process memory
ps aux | grep uvicorn

# Check in-memory storage size
curl http://localhost:8000/api/v1/calls | jq length

# Check ngrok process
ps aux | grep ngrok

# Check ngrok logs
tail -f ${NGROK_LOG_FILE:-/tmp/ngrok.log}

# app/core/logging.py
import logging

def setup_logging():
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(message)s"
    )

# Call lifecycle
grep "\[telephony\]" logs.txt

# Audio streaming
grep "streaming_start" logs.txt

# Transcription
grep "\[stt\]" logs.txt

# Emotion analysis
grep "\[emotion\]" logs.txt

# Triage decisions
grep "\[triage:minimal\]" logs.txt

# Summary generation
grep "\[summary\]" logs.txt

# Save logs to file
./scripts/dev_start.sh 2>&1 | tee logs.txt

# Filter errors only
./scripts/dev_start.sh 2>&1 | grep -i error

# JSON formatted logs (future)
# Configure structured logging in app/core/logging.py

pip install -U pip
pip install -r requirements.txt --upgrade

version: "2"
tunnels:
  api:
    addr: 8000
    proto: http
    inspect: false
  ws:
    addr: 8000
    proto: http
    inspect: false

# All queues and calls cleared
./scripts/dev_start.sh

# General rule: (2 x cores) + 1
gunicorn app.main:app \
  --workers $((2 * $(nproc) + 1)) \
  --worker-class uvicorn.workers.UvicornWorker

# Telnyx API calls
async with httpx.AsyncClient(timeout=8) as c:

# Deepgram transcription
async with httpx.AsyncClient(timeout=30) as c:

# OpenAI/Deepgram emotion
async with httpx.AsyncClient(timeout=10) as c:

# Future: app/core/database.py
engine = create_async_engine(
    DATABASE_URL,
    pool_size=20,
    max_overflow=10,
    pool_timeout=30,
)