Monitoring and Observability

pnpm add @opentelemetry/sdk-node \
         @opentelemetry/auto-instrumentations-node \
         @opentelemetry/exporter-trace-otlp-grpc \
         @opentelemetry/exporter-metrics-otlp-grpc

import { trace } from '@opentelemetry/api';

const tracer = trace.getTracer('fluxer-server');

async function sendMessage(channelId: string, content: string) {
  const span = tracer.startSpan('sendMessage');
  span.setAttributes({
    'channel.id': channelId,
    'message.length': content.length,
  });
  
  try {
    // Validate content
    await validateMessage(content);
    
    // Save to Cassandra
    const message = await db.messages.insert({
      channelId,
      content,
      timestamp: Date.now(),
    });
    
    // Publish to NATS
    await nats.publish(`channel.${channelId}.message`, message);
    
    span.setStatus({ code: SpanStatusCode.OK });
    return message;
  } catch (error) {
    span.recordException(error);
    span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
    throw error;
  } finally {
    span.end();
  }
}

import { metrics } from '@opentelemetry/api';

const meter = metrics.getMeter('fluxer-server');

const messageCounter = meter.createCounter('fluxer.messages.sent', {
  description: 'Total messages sent',
  unit: '1',
});

const messageHistogram = meter.createHistogram('fluxer.message.size', {
  description: 'Message size distribution',
  unit: 'bytes',
});

// Record metrics
messageCounter.add(1, {
  'channel.type': 'text',
  'guild.id': guildId,
});

messageHistogram.record(content.length, {
  'channel.type': 'text',
});

services:
  fluxer_api:
    deploy:
      labels:
        signoz.io/scrape: 'true'
        signoz.io/port: '9464'
        signoz.io/path: '/metrics'

import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import express from 'express';

const prometheusExporter = new PrometheusExporter({
  port: 9464,
  endpoint: '/metrics',
});

const app = express();
// ... app routes

app.listen(8080);

import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  transport: {
    target: 'pino-opentelemetry-transport',
    options: {
      exporterUrl: 'http://otel-collector:4318/v1/logs',
    },
  },
});

logger.info({ userId, channelId }, 'User sent message');
logger.error({ error: err, userId }, 'Failed to send message');

-- Find all errors for user
service_name = 'fluxer-server' AND level = 'error' AND userId = '123456'

-- Find slow database queries
service_name = 'fluxer-server' AND attributes.db.duration > 1000

-- Find specific error messages
service_name = 'fluxer-api' AND body CONTAINS 'rate limit exceeded'

processors:
  tail_sampling:
    policies:
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      
      - name: slow-requests
        type: latency
        latency:
          threshold_ms: 1000
      
      - name: sample-others
        type: probabilistic
        probabilistic:
          sampling_percentage: 10