Observability and the Three Pillars: Logs, Metrics, and Traces

// Bad: Unstructured log// Good: Structured log// Good: Structured log// Good: Structured loguctured log
logger.info("purchase_completed", {
  user_id: "123",
  item_id: "456",
  amount: 99.99,
  currency: "USD",
  trace_id: "abc123def456",
  span_id: "789xyz",
  timestamp: "2025-01-14T10:30:00Z"
});

# Fluent Bit configuration for Kubernetes# fluent-bit.conff
[SERVICE]
    Flush        1
    Log_Level    info
    Parsers_File parsers.conf

[INPUT]
    Name             tail
    Path             /var/log/containers/*.log
    Parser           docker
    Tag              kube.*
    Refresh_Interval 5
    Mem_Buf_Limit    5MB

[FILTER]
    Name                kubernetes
    Match               kube.*
    Kube_URL            https://kubernetes.default.svc:443
    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
    Merge_Log           On
    K8S-Logging.Parser  On
    K8S-Logging.Exclude On

[OUTPUT]
    Name            loki
    Match           *
    Host            loki.monitoring.svc.cluster.local
    Port            3100
    Labels          job=fluent-bit
    Auto_Kubernetes_Labels on

# Application metrics with Prometheus client// Request counter// Request counter// Request counter// Request counter// Request counter// Request counter// Request duration histogram// Request duration histogram// Request duration histogram// Request duration histogram// Request duration histogram// Request duration histogram// Request duration histogram// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metrics// Middleware to record metricsd metrics
app.use((req, res, next) => {
  const start = Date.now();
  
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    
    httpRequestsTotal.inc({
      method: req.method,
      path: req.route?.path || req.path,
      status: res.statusCode,
    });
    
    httpRequestDuration.observe(
      { method: req.method, path: req.route?.path || req.path },
      duration
    );
  });
  
  next();
});

# Request rate per second (last 5 minutes)# Error rate percentage# Error rate percentage# 95th percentile latency# 95th percentile latency# 95th percentile latency# 95th percentile latency# 95th percentile latency# Requests by service# Requests by service# Requests by service# Requests by service# Requests by serviceRequests by service
sum by (service) (rate(http_requests_total[5m]))

# traceparent header format# tracestate for vendor-specific data# tracestate for vendor-specific data# tracestate for vendor-specific data# tracestate for vendor-specific data data
tracestate: vendor1=value1,vendor2=value2

// tracing.ts - Initialize before importing other modules
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'my-api',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
    [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: 'production',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4318/v1/metrics',
    }),
    exportIntervalMillis: 60000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingPaths: ['/health', '/ready'],
      },
      '@opentelemetry/instrumentation-fs': { enabled: false },
    }),
  ],
});

sdk.start();

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 10s
    send_batch_size: 1000
  
  memory_limiter:
    check_interval: 1s
    limit_mib: 1000
    spike_limit_mib: 200
  
  resource:
    attributes:
      - key: environment
        value: production
        action: upsert

exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    
  jaeger:
    endpoint: jaeger:14250
    tls:
      insecure: true
  
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

# For 99.9% availability SLO over 30 days:# In terms of downtime:# In terms of downtime:# Prometheus query for remaining error budget# Prometheus query for remaining error budget# Prometheus query for remaining error budgetPrometheus query for remaining error budget
1 - (
  sum(rate(http_requests_total{status=~"5.."}[30d])) 
  / sum(rate(http_requests_total[30d]))
) / 0.001  # 0.1% error budget

# Tail-based sampling in OTel Collector
processors:
  tail_sampling:
    decision_wait: 10s
    policies:
      # Always sample errors
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      # Sample slow requests
      - name: slow-requests
        type: latency
        latency:
          threshold_ms: 1000
      # Sample 10% of everything else
      - name: probabilistic
        type: probabilistic
        probabilistic:
          sampling_percentage: 10