🔨 Add OTel/Jeager to start HTTP/SQL requests monitoring

backend/backend/db/db.py

@@ -1,3 +1,4 @@
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import (
     async_sessionmaker,
@@ -26,6 +27,8 @@ class Database:
         # TODO: Preserve UserLastStopSearchResults table from drop.
         self._engine = create_async_engine(db_path)
         if self._engine is not None:
+            SQLAlchemyInstrumentor().instrument(engine=self._engine.sync_engine)
+
             self._session_maker = async_sessionmaker(
                 self._engine, expire_on_commit=False, class_=AsyncSession
             )

backend/docker-compose.yml

@@ -13,7 +13,62 @@ services:
         max-size: 10m
         max-file: "3"
     ports:
-      - '127.0.0.1:5438:5432'
+      - "127.0.0.1:5432:5432"
     volumes:
       - ./docker/database/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       - ./docker/database/data:/var/lib/postgresql/data
+
+  jaeger-agent:
+    image: jaegertracing/jaeger-agent:latest
+    command:
+      - "--reporter.grpc.host-port=jaeger-collector:14250"
+    ports:
+      - "127.0.0.1:5775:5775/udp"
+      - "127.0.0.1:6831:6831/udp"
+      - "127.0.0.1:6832:6832/udp"
+      - "127.0.0.1:5778:5778"
+    restart: on-failure
+    depends_on:
+      - jaeger-collector
+
+  jaeger-collector:
+    image: jaegertracing/jaeger-collector:latest
+    command:
+      - "--cassandra.keyspace=jaeger_v1_dc1"
+      - "--cassandra.servers=cassandra"
+      - "--collector.zipkin.host-port=9411"
+      - "--sampling.initial-sampling-probability=.5"
+      - "--sampling.target-samples-per-second=.01"
+    environment:
+      - SAMPLING_CONFIG_TYPE=adaptive
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - "127.0.0.1:4317:4317"
+      - "127.0.0.1:4318:4318"
+      # - "127.0.0.1:9411:9411"
+      # - "127.0.0.1:14250:14250"
+      # - "127.0.0.1:14268:14268"
+      # - "127.0.0.1:14269:14269"
+    restart: on-failure
+    depends_on:
+      - cassandra-schema
+
+  cassandra:
+    image: cassandra:latest
+
+  cassandra-schema:
+    image: jaegertracing/jaeger-cassandra-schema:latest
+    depends_on:
+      - cassandra
+
+  jaeger-query:
+    image: jaegertracing/jaeger-query:latest
+    command:
+      - "--cassandra.keyspace=jaeger_v1_dc1"
+      - "--cassandra.servers=cassandra"
+    ports:
+      - "127.0.0.1:16686:16686"
+      # - "127.0.0.1:16687:16687"
+    restart: on-failure
+    depends_on:
+      - cassandra-schema

backend/main.py

@@ -1,12 +1,26 @@
+#!/usr/bin/env python3
+import logging
 from collections import defaultdict
 from datetime import datetime
 from os import environ, EX_USAGE
 from typing import Sequence
 
+import uvicorn
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource as OtResource
+from opentelemetry.sdk.trace import TracerProvider as OtTracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from rich import print
+from starlette.types import ASGIApp
 
 from backend.db import db
 from backend.idfm_interface import Destinations as IdfmDestinations, IdfmInterface
@@ -26,6 +40,11 @@ if API_KEY is None:
     print('No "API_KEY" environment variable set... abort.')
     exit(EX_USAGE)
 
+APP_NAME = environ.get("APP_NAME", "app")
+MODE = environ.get("MODE", "grpc")
+COLLECTOR_ENDPOINT_GRPC_ENDPOINT = environ.get(
+    "COLLECTOR_ENDPOINT_GRPC_ENDPOINT", "127.0.0.1:14250"  # "jaeger-collector:14250"
+)
 # TODO: Remove postgresql+asyncpg from environ variable
 DB_PATH = "postgresql+asyncpg://cer_user:cer_password@127.0.0.1:5438/cer_db"
 
@@ -42,6 +61,14 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+trace.set_tracer_provider(TracerProvider())
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+tracer = trace.get_tracer(APP_NAME)
+
+with tracer.start_as_current_span("foo"):
+    print("Hello world!")
+
+
 idfm_interface = IdfmInterface(API_KEY, db)
 
 
@@ -247,3 +274,9 @@ async def get_stop_shape(stop_id: int) -> StopShapeSchema | None:
 
     msg = f"No shape found for stop {stop_id}"
     raise HTTPException(status_code=404, detail=msg)
+
+
+FastAPIInstrumentor.instrument_app(app)
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=4443, ssl_certfile="./config/cert.pem")

backend/pyproject.toml

@@ -18,7 +18,13 @@ asyncpg = "^0.27.0"
 msgspec = "^0.12.0"
 pyshp = "^2.3.1"
 pyproj = "^3.5.0"
+opentelemetry-instrumentation-fastapi = "^0.38b0"
 sqlalchemy-utils = "^0.41.1"
+opentelemetry-instrumentation-logging = "^0.38b0"
+opentelemetry-sdk = "^1.17.0"
+opentelemetry-api = "^1.17.0"
+opentelemetry-exporter-otlp-proto-http = "^1.17.0"
+opentelemetry-instrumentation-sqlalchemy = "^0.38b0"
 
 [build-system]
 requires = ["poetry-core"]