From e2ff90cd5f4fe826f148ef26b605e23ac873386a Mon Sep 17 00:00:00 2001 From: Adrien Date: Fri, 26 May 2023 18:10:47 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Use=20Redis=20to=20cache?= =?UTF-8?q?=20REST=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/backend/settings.py | 37 +++++++++++++++++++++++++++++++++++-- backend/dependencies.py | 18 +++++++++++++++++- backend/main.py | 30 +++++++++++++++++++++--------- backend/pyproject.toml | 1 + backend/routers/line.py | 2 ++ backend/routers/stop.py | 6 +++++- docker-compose.yml | 6 ++++++ 7 files changed, 87 insertions(+), 13 deletions(-) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 356e6b6..71579e7 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -1,4 +1,6 @@ -from pydantic import BaseModel, BaseSettings, Field, SecretStr +from typing import Any + +from pydantic import BaseModel, BaseSettings, Field, root_validator, SecretStr class HttpSettings(BaseModel): @@ -7,14 +9,43 @@ class HttpSettings(BaseModel): cert: str | None = None +def check_user_password(cls, values: dict[str, Any]) -> dict[str, Any]: + user = values.get("user") + password = values.get("password") + + if user is not None and password is None: + raise ValueError("user is set, password shall be set too.") + + if password is not None and user is None: + raise ValueError("password is set, user shall be set too.") + + return values + + class DatabaseSettings(BaseModel): name: str = "carrramba-encore-rate" host: str = "127.0.0.1" port: int = 5432 driver: str = "postgresql+psycopg" - user: str = "cer" + user: str | None = None password: SecretStr | None = None + _user_password_validation = root_validator(allow_reuse=True)(check_user_password) + + +class CacheSettings(BaseModel): + enable: bool = False + host: str = "127.0.0.1" + port: int = 6379 + user: str | None = None + password: SecretStr | None = None + + _user_password_validation = root_validator(allow_reuse=True)(check_user_password) + + +class TracingSettings(BaseModel): + enable: bool = False + class Settings(BaseSettings): app_name: str @@ -24,3 +55,5 @@ class Settings(BaseSettings): http: HttpSettings = HttpSettings() db: DatabaseSettings = DatabaseSettings() + cache: CacheSettings = CacheSettings() + tracing: TracingSettings = TracingSettings() diff --git a/backend/dependencies.py b/backend/dependencies.py index c2ee1f6..02aaf49 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -1,10 +1,12 @@ from os import environ +from fastapi_cache.backends.redis import RedisBackend +from redis import asyncio as aioredis from yaml import safe_load from backend.db import db from backend.idfm_interface import IdfmInterface -from backend.settings import Settings +from backend.settings import CacheSettings, Settings CONFIG_PATH = environ.get("CONFIG_PATH", "./config.sample.yaml") @@ -20,3 +22,17 @@ def load_settings(path: str) -> Settings: settings = load_settings(CONFIG_PATH) idfm_interface = IdfmInterface(settings.idfm_api_key.get_secret_value(), db) + + +def init_redis_backend(settings: CacheSettings) -> RedisBackend: + login = f"{settings.user}:{settings.password}@" if settings.user is not None else "" + + url = f"redis://{login}{settings.host}:{settings.port}" + + redis_connections_pool = aioredis.from_url( + url, encoding="utf8", decode_responses=True + ) + return RedisBackend(redis_connections_pool) + + +redis_backend = init_redis_backend(settings.cache) diff --git a/backend/main.py b/backend/main.py index 3ae627f..b27bd7c 100755 --- a/backend/main.py +++ b/backend/main.py @@ -5,6 +5,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles +from fastapi_cache import FastAPICache from opentelemetry import trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -13,12 +14,14 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from backend.db import db -from dependencies import idfm_interface, settings +from dependencies import idfm_interface, redis_backend, settings from routers import line, stop @asynccontextmanager async def lifespan(app: FastAPI): + FastAPICache.init(redis_backend, prefix="api", enable=settings.cache.enable) + await db.connect(settings.db, settings.clear_static_data) if settings.clear_static_data: await idfm_interface.startup() @@ -44,20 +47,29 @@ app.include_router(line.router) app.include_router(stop.router) -FastAPIInstrumentor.instrument_app(app) +if settings.tracing.enable: + FastAPIInstrumentor.instrument_app(app) -trace.set_tracer_provider( - TracerProvider(resource=Resource.create({SERVICE_NAME: settings.app_name})) -) -trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) -tracer = trace.get_tracer(settings.app_name) + trace.set_tracer_provider( + TracerProvider(resource=Resource.create({SERVICE_NAME: settings.app_name})) + ) + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) + ) + tracer = trace.get_tracer(settings.app_name) if __name__ == "__main__": http_settings = settings.http - uvicorn.run( - app, + + config = uvicorn.Config( + app=app, host=http_settings.host, port=http_settings.port, ssl_certfile=http_settings.cert, + proxy_headers=True, ) + + server = uvicorn.Server(config) + + server.run() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8a012e7..e104d15 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -25,6 +25,7 @@ opentelemetry-instrumentation-sqlalchemy = "^0.38b0" sqlalchemy = "^2.0.12" psycopg = "^3.1.9" pyyaml = "^6.0" +fastapi-cache2 = {extras = ["redis"], version = "^0.2.1"} [build-system] requires = ["poetry-core"] diff --git a/backend/routers/line.py b/backend/routers/line.py index 1c8463f..917881f 100644 --- a/backend/routers/line.py +++ b/backend/routers/line.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, HTTPException +from fastapi_cache.decorator import cache from backend.models import Line from backend.schemas import Line as LineSchema, TransportMode @@ -8,6 +9,7 @@ router = APIRouter(prefix="/line", tags=["line"]) @router.get("/{line_id}", response_model=LineSchema) +@cache(namespace="line") async def get_line(line_id: int) -> LineSchema: line: Line | None = await Line.get_by_id(line_id) diff --git a/backend/routers/stop.py b/backend/routers/stop.py index 1f3e803..a0cf645 100644 --- a/backend/routers/stop.py +++ b/backend/routers/stop.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Sequence from fastapi import APIRouter, HTTPException +from fastapi_cache.decorator import cache from backend.idfm_interface import ( Destinations as IdfmDestinations, @@ -40,6 +41,7 @@ def optional_datetime_to_ts(dt: datetime | None) -> int | None: # TODO: Add limit support @router.get("/") +@cache(namespace="stop") async def get_stop( name: str = "", limit: int = 10 ) -> Sequence[StopAreaSchema | StopSchema] | None: @@ -83,8 +85,8 @@ async def get_stop( return formatted -# TODO: Cache response for 30 secs ? @router.get("/{stop_id}/nextPassages") +@cache(namespace="stop-nextPassages", expire=30) async def get_next_passages(stop_id: int) -> NextPassagesSchema | None: res = await idfm_interface.get_next_passages(stop_id) if res is None: @@ -149,6 +151,7 @@ async def get_next_passages(stop_id: int) -> NextPassagesSchema | None: @router.get("/{stop_id}/destinations") +@cache(namespace="stop-destinations", expire=30) async def get_stop_destinations( stop_id: int, ) -> IdfmDestinations | None: @@ -158,6 +161,7 @@ async def get_stop_destinations( @router.get("/{stop_id}/shape") +@cache(namespace="stop-shape") async def get_stop_shape(stop_id: int) -> StopShapeSchema | None: connection_area = None diff --git a/docker-compose.yml b/docker-compose.yml index feb0d68..6a44abc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,12 @@ services: - ./backend/docker/database/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - ./backend/docker/database/data:/var/lib/postgresql/data + redis: + image: redis:latest + restart: always + command: redis-server --loglevel warning + ports: + - "127.0.0.1:6379:6379" jaeger-agent: image: jaegertracing/jaeger-agent:latest