♻️ Use of pydantic to manage config+env variables

FastAPI release has been updated allowing to use lifespan parameter
to prepare/shutdown sub components.

backend/backend/db/base_class.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import Iterable, Self, TYPE_CHECKING
 
 from sqlalchemy import select
@@ -9,31 +10,36 @@ from sqlalchemy.orm import DeclarativeBase
 if TYPE_CHECKING:
     from .db import Database
 
+logger = getLogger(__name__)
+
 
 class Base(DeclarativeBase):
     db: Database | None = None
 
     @classmethod
-    async def add(cls, stops: Self | Iterable[Self]) -> bool:
+    async def add(cls, objs: Self | Iterable[Self]) -> bool:
-        try:
+        if cls.db is not None and (session := await cls.db.get_session()) is not None:
-            if isinstance(stops, Iterable):
+
-                cls.db.session.add_all(stops)  # type: ignore
+            async with session.begin():
-            else:
+                try:
-                cls.db.session.add(stops)  # type: ignore
+                    if isinstance(objs, Iterable):
-            await cls.db.session.commit()  # type: ignore
+                        session.add_all(objs)
-        except (AttributeError, IntegrityError) as err:
+                    else:
-            print(err)
+                        session.add(objs)
-            return False
+
+                except (AttributeError, IntegrityError) as err:
+                    logger.error(err)
+                    return False
 
         return True
 
     @classmethod
     async def get_by_id(cls, id_: int | str) -> Self | None:
-        try:
+        if cls.db is not None and (session := await cls.db.get_session()) is not None:
-            stmt = select(cls).where(cls.id == id_)  # type: ignore
+
-            res = await cls.db.session.execute(stmt)  # type: ignore
+            async with session.begin():
-            element = res.scalar_one_or_none()
+                stmt = select(cls).where(cls.id == id_)
-        except AttributeError as err:
+                res = await session.execute(stmt)
-            print(err)
+                return res.scalar_one_or_none()
-            element = None
+
-        return element
+        return None

backend/backend/db/db.py

@@ -1,4 +1,10 @@
+from logging import getLogger
+from typing import Annotated, AsyncIterator
+
+from fastapi import Depends
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 from sqlalchemy import text
+from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import (
     async_sessionmaker,
     AsyncEngine,
@@ -7,42 +13,57 @@ from sqlalchemy.ext.asyncio import (
 )
 
 from .base_class import Base
+from ..settings import DatabaseSettings
+
+logger = getLogger(__name__)
 
 
 class Database:
     def __init__(self) -> None:
-        self._engine: AsyncEngine | None = None
+        self._async_engine: AsyncEngine | None = None
-        self._session_maker: async_sessionmaker[AsyncSession] | None = None
+        self._async_session_local: async_sessionmaker[AsyncSession] | None = None
-        self._session: AsyncSession | None = None
 
-    @property
+    async def get_session(self) -> AsyncSession | None:
-    def session(self) -> AsyncSession | None:
+        try:
-        if self._session is None and (session_maker := self._session_maker) is not None:
+            return self._async_session_local()  # type: ignore
-            self._session = session_maker()
-        return self._session
 
-    async def connect(self, db_path: str, clear_static_data: bool = False) -> bool:
+        except (SQLAlchemyError, AttributeError) as e:
+            logger.exception(e)
 
-        # TODO: Preserve UserLastStopSearchResults table from drop.
+        return None
-        self._engine = create_async_engine(db_path)
+
-        if self._engine is not None:
+    # TODO: Preserve UserLastStopSearchResults table from drop.
-            self._session_maker = async_sessionmaker(
+    async def connect(
-                self._engine, expire_on_commit=False, class_=AsyncSession
+        self, settings: DatabaseSettings, clear_static_data: bool = False
+    ) -> bool:
+        password = settings.password
+        path = (
+            f"{settings.driver}://{settings.user}:"
+            f"{password.get_secret_value() if password is not None else ''}"
+            f"@{settings.host}:{settings.port}/{settings.name}"
+        )
+        self._async_engine = create_async_engine(
+            path, pool_pre_ping=True, pool_size=10, max_overflow=20
+        )
+
+        if self._async_engine is not None:
+            SQLAlchemyInstrumentor().instrument(engine=self._async_engine.sync_engine)
+
+            self._async_session_local = async_sessionmaker(
+                bind=self._async_engine,
+                # autoflush=False,
+                expire_on_commit=False,
+                class_=AsyncSession,
             )
-            if (session := self.session) is not None:
-                await session.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;"))
 
-            async with self._engine.begin() as conn:
+            async with self._async_engine.begin() as session:
+                await session.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;"))
                 if clear_static_data:
-                    await conn.run_sync(Base.metadata.drop_all)
+                    await session.run_sync(Base.metadata.drop_all)
-                await conn.run_sync(Base.metadata.create_all)
+                    await session.run_sync(Base.metadata.create_all)
 
         return True
 
     async def disconnect(self) -> None:
-        if self._session is not None:
+        if self._async_engine is not None:
-            await self._session.close()
+            await self._async_engine.dispose()
-            self._session = None
-
-        if self._engine is not None:
-            await self._engine.dispose()

backend/backend/idfm_interface/idfm_interface.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from logging import getLogger
 from re import compile as re_compile
 from time import time
 from typing import (
@@ -15,7 +16,7 @@ from aiohttp import ClientSession
 from msgspec import ValidationError
 from msgspec.json import Decoder
 from pyproj import Transformer
-from shapefile import Reader as ShapeFileReader, ShapeRecord
+from shapefile import Reader as ShapeFileReader, ShapeRecord  # type: ignore
 
 from ..db import Database
 from ..models import ConnectionArea, Line, LinePicto, Stop, StopArea, StopShape
@@ -37,6 +38,9 @@ from .idfm_types import (
 from .ratp_types import Picto as RatpPicto
 
 
+logger = getLogger(__name__)
+
+
 class IdfmInterface:
 
     IDFM_ROOT_URL = "https://prim.iledefrance-mobilites.fr/marketplace"
@@ -55,7 +59,7 @@ class IdfmInterface:
     )
 
     OPERATOR_RE = re_compile(r"[^:]+:Operator::([^:]+):")
-    LINE_RE = re_compile(r"[^:]+:Line::([^:]+):")
+    LINE_RE = re_compile(r"[^:]+:Line::C([^:]+):")
 
     def __init__(self, api_key: str, database: Database) -> None:
         self._api_key = api_key
@@ -357,11 +361,23 @@ class IdfmInterface:
             fields = line.fields
 
             picto_id = fields.picto.id_ if fields.picto is not None else None
-            picto = await LinePicto.get_by_id(picto_id) if picto_id else None
+
+            line_id = fields.id_line
+            try:
+                formatted_line_id = int(line_id[1:] if line_id[0] == "C" else line_id)
+            except ValueError:
+                logger.warning("Unable to format %s line id.", line_id)
+                continue
+
+            try:
+                operator_id = int(fields.operatorref)  # type: ignore
+            except (ValueError, TypeError):
+                logger.warning("Unable to format %s operator id.", fields.operatorref)
+                operator_id = 0
 
             ret.append(
                 Line(
-                    id=fields.id_line,
+                    id=formatted_line_id,
                     short_name=fields.shortname_line,
                     name=fields.name_line,
                     status=IdfmLineState(fields.status.value),
@@ -374,7 +390,7 @@ class IdfmInterface:
                     ),
                     colour_web_hexa=fields.colourweb_hexa,
                     text_colour_hexa=fields.textcolourprint_hexa,
-                    operator_id=optional_value(fields.operatorref),
+                    operator_id=operator_id,
                     operator_name=optional_value(fields.operatorname),
                     accessibility=IdfmState(fields.accessibility.value),
                     visual_signs_available=IdfmState(
@@ -384,7 +400,6 @@ class IdfmInterface:
                         fields.audiblesigns_available.value
                     ),
                     picto_id=fields.picto.id_ if fields.picto is not None else None,
-                    picto=picto,
                     record_id=line.recordid,
                     record_ts=int(line.record_timestamp.timestamp()),
                 )
@@ -405,13 +420,21 @@ class IdfmInterface:
                 fields.arrxepsg2154, fields.arryepsg2154
             )
 
+            try:
+                postal_region = int(fields.arrpostalregion)
+            except ValueError:
+                logger.warning(
+                    "Unable to format %s postal region.", fields.arrpostalregion
+                )
+                continue
+
             yield Stop(
                 id=int(fields.arrid),
                 name=fields.arrname,
                 epsg3857_x=epsg3857_point[0],
                 epsg3857_y=epsg3857_point[1],
                 town_name=fields.arrtown,
-                postal_region=fields.arrpostalregion,
+                postal_region=postal_region,
                 transport_mode=TransportMode(fields.arrtype.value),
                 version=fields.arrversion,
                 created_ts=created_ts,

backend/backend/models/line.py

@@ -5,13 +5,11 @@ from typing import Iterable, Self, Sequence
 from sqlalchemy import (
     BigInteger,
     Boolean,
-    Column,
     Enum,
     ForeignKey,
     Integer,
     select,
     String,
-    Table,
 )
 from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload
 from sqlalchemy.sql.expression import tuple_
@@ -25,12 +23,14 @@ from ..idfm_interface.idfm_types import (
 )
 from .stop import _Stop
 
-line_stop_association_table = Table(
+
-    "line_stop_association_table",
+class LineStopAssociations(Base):
-    Base.metadata,
+
-    Column("line_id", ForeignKey("lines.id")),
+    id = mapped_column(BigInteger, primary_key=True)
-    Column("stop_id", ForeignKey("_stops.id")),
+    line_id = mapped_column(BigInteger, ForeignKey("lines.id"))
-)
+    stop_id = mapped_column(BigInteger, ForeignKey("_stops.id"))
+
+    __tablename__ = "line_stop_associations"
 
 
 class LinePicto(Base):
@@ -53,7 +53,7 @@ class Line(Base):
 
     db = db
 
-    id = mapped_column(String, primary_key=True)
+    id = mapped_column(BigInteger, primary_key=True)
 
     short_name = mapped_column(String)
     name = mapped_column(String, nullable=False)
@@ -68,7 +68,7 @@ class Line(Base):
     colour_web_hexa = mapped_column(String, nullable=False)
     text_colour_hexa = mapped_column(String, nullable=False)
 
-    operator_id = mapped_column(String)
+    operator_id = mapped_column(Integer)
     operator_name = mapped_column(String)
 
     accessibility = mapped_column(Enum(IdfmState), nullable=False)
@@ -83,7 +83,7 @@ class Line(Base):
 
     stops: Mapped[list[_Stop]] = relationship(
         "_Stop",
-        secondary=line_stop_association_table,
+        secondary="line_stop_associations",
         back_populates="lines",
         lazy="selectin",
     )
@@ -94,23 +94,24 @@ class Line(Base):
     async def get_by_name(
         cls, name: str, operator_name: None | str = None
     ) -> Sequence[Self] | None:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return None
 
-        filters = {"name": name}
+            async with session.begin():
-        if operator_name is not None:
+                filters = {"name": name}
-            filters["operator_name"] = operator_name
+                if operator_name is not None:
+                    filters["operator_name"] = operator_name
 
-        stmt = (
+                stmt = (
-            select(cls)
+                    select(cls)
-            .filter_by(**filters)
+                    .filter_by(**filters)
-            .options(selectinload(cls.stops), selectinload(cls.picto))
+                    .options(selectinload(cls.stops), selectinload(cls.picto))
-        )
+                )
-        res = await session.execute(stmt)
+                res = await session.execute(stmt)
-        lines = res.scalars().all()
+                lines = res.scalars().all()
 
-        return lines
+                return lines
+
+        return None
 
     @classmethod
     async def _add_picto_to_line(cls, line: str | Self, picto: LinePicto) -> None:
@@ -133,57 +134,63 @@ class Line(Base):
 
     @classmethod
     async def add_pictos(cls, line_to_pictos: Iterable[tuple[str, LinePicto]]) -> bool:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return False
 
-        await asyncio_gather(
+            async with session.begin():
-            *[cls._add_picto_to_line(line, picto) for line, picto in line_to_pictos]
+                await asyncio_gather(
-        )
+                    *[
+                        cls._add_picto_to_line(line, picto)
+                        for line, picto in line_to_pictos
+                    ]
+                )
 
-        await session.commit()
+                return True
 
-        return True
+        return False
 
     @classmethod
     async def add_stops(cls, line_to_stop_ids: Iterable[tuple[str, str, int]]) -> int:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return 0
 
-        line_names_ops, stop_ids = set(), set()
+            async with session.begin():
-        for line_name, operator_name, stop_id in line_to_stop_ids:
-            line_names_ops.add((line_name, operator_name))
-            stop_ids.add(stop_id)
 
-        lines_res = await session.execute(
+                line_names_ops, stop_ids = set(), set()
-            select(Line).where(
+                for line_name, operator_name, stop_id in line_to_stop_ids:
-                tuple_(Line.name, Line.operator_name).in_(line_names_ops)
+                    line_names_ops.add((line_name, operator_name))
-            )
+                    stop_ids.add(stop_id)
-        )
 
-        lines = defaultdict(list)
+                lines_res = await session.execute(
-        for line in lines_res.scalars():
+                    select(Line).where(
-            lines[(line.name, line.operator_name)].append(line)
+                        tuple_(Line.name, Line.operator_name).in_(line_names_ops)
-
+                    )
-        stops_res = await session.execute(select(_Stop).where(_Stop.id.in_(stop_ids)))
-        stops = {stop.id: stop for stop in stops_res.scalars()}
-
-        found = 0
-        for line_name, operator_name, stop_id in line_to_stop_ids:
-            if (stop := stops.get(stop_id)) is not None:
-                if (stop_lines := lines.get((line_name, operator_name))) is not None:
-                    for stop_line in stop_lines:
-                        stop_line.stops.append(stop)
-                    found += 1
-                else:
-                    print(f"No line found for {line_name}/{operator_name}")
-            else:
-                print(
-                    f"No stop found for {stop_id} id"
-                    f"(used by {line_name}/{operator_name})"
                 )
 
-        await session.commit()
+                lines = defaultdict(list)
+                for line in lines_res.scalars():
+                    lines[(line.name, line.operator_name)].append(line)
 
-        return found
+                stops_res = await session.execute(
+                    select(_Stop).where(_Stop.id.in_(stop_ids))
+                )
+                stops = {stop.id: stop for stop in stops_res.scalars()}
+
+                found = 0
+                for line_name, operator_name, stop_id in line_to_stop_ids:
+                    if (stop := stops.get(stop_id)) is not None:
+                        if (
+                            stop_lines := lines.get((line_name, operator_name))
+                        ) is not None:
+                            for stop_line in stop_lines:
+                                stop_line.stops.append(stop)
+                            found += 1
+                        else:
+                            print(f"No line found for {line_name}/{operator_name}")
+                    else:
+                        print(
+                            f"No stop found for {stop_id} id"
+                            f"(used by {line_name}/{operator_name})"
+                        )
+
+                return found
+
+        return 0

backend/backend/models/stop.py

@@ -1,18 +1,20 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import Iterable, Sequence, TYPE_CHECKING
 
 from sqlalchemy import (
     BigInteger,
-    Column,
+    Computed,
+    desc,
     Enum,
     Float,
     ForeignKey,
+    func,
     Integer,
     JSON,
     select,
     String,
-    Table,
 )
 from sqlalchemy.orm import (
     mapped_column,
@@ -22,6 +24,7 @@ from sqlalchemy.orm import (
     with_polymorphic,
 )
 from sqlalchemy.schema import Index
+from sqlalchemy_utils.types.ts_vector import TSVectorType
 
 from ..db import Base, db
 from ..idfm_interface.idfm_types import TransportMode, IdfmState, StopAreaType
@@ -30,12 +33,15 @@ if TYPE_CHECKING:
     from .line import Line
 
 
-stop_area_stop_association_table = Table(
+logger = getLogger(__name__)
-    "stop_area_stop_association_table",
+
-    Base.metadata,
+class StopAreaStopAssociations(Base):
-    Column("stop_id", ForeignKey("_stops.id")),
+
-    Column("stop_area_id", ForeignKey("stop_areas.id")),
+    id = mapped_column(BigInteger, primary_key=True)
-)
+    stop_id = mapped_column(BigInteger, ForeignKey("_stops.id"))
+    stop_area_id = mapped_column(BigInteger, ForeignKey("stop_areas.id"))
+
+    __tablename__ = "stop_area_stop_associations"
 
 
 class _Stop(Base):
@@ -47,7 +53,7 @@ class _Stop(Base):
 
     name = mapped_column(String, nullable=False, index=True)
     town_name = mapped_column(String, nullable=False)
-    postal_region = mapped_column(String, nullable=False)
+    postal_region = mapped_column(Integer, nullable=False)
     epsg3857_x = mapped_column(Float, nullable=False)
     epsg3857_y = mapped_column(Float, nullable=False)
 
@@ -57,12 +63,14 @@ class _Stop(Base):
 
     lines: Mapped[list[Line]] = relationship(
         "Line",
-        secondary="line_stop_association_table",
+        secondary="line_stop_associations",
         back_populates="stops",
         lazy="selectin",
     )
     areas: Mapped[list["StopArea"]] = relationship(
-        "StopArea", secondary=stop_area_stop_association_table, back_populates="stops"
+        "StopArea",
+        secondary="stop_area_stop_associations",
+        back_populates="stops",
     )
     connection_area_id: Mapped[int] = mapped_column(
         ForeignKey("connection_areas.id"), nullable=True
@@ -71,40 +79,45 @@ class _Stop(Base):
         back_populates="stops", lazy="selectin"
     )
 
+    names_tsv = mapped_column(
+        TSVectorType("name", "town_name", regconfig="french"),
+        Computed("to_tsvector('french', name || ' ' || town_name)", persisted=True),
+    )
+
     __tablename__ = "_stops"
     __mapper_args__ = {"polymorphic_identity": "_stops", "polymorphic_on": kind}
     __table_args__ = (
-        # To optimize the ilike requests
         Index(
-            "name_idx_gin",
+            "names_tsv_idx",
-            name,
+            names_tsv,
             postgresql_ops={"name": "gin_trgm_ops"},
             postgresql_using="gin",
         ),
     )
 
-    # TODO: Test https://www.cybertec-postgresql.com/en/postgresql-more-performance-for-like-and-ilike-statements/
-    # TODO: Should be able to remove with_polymorphic ?
     @classmethod
-    async def get_by_name(cls, name: str) -> Sequence[type[_Stop]] | None:
+    async def get_by_name(cls, name: str) -> Sequence[_Stop] | None:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return None
 
-        stop_stop_area = with_polymorphic(_Stop, [Stop, StopArea])
+            async with session.begin():
-        stmt = (
+                descendants = with_polymorphic(_Stop, "*")
-            select(stop_stop_area)
-            .where(stop_stop_area.name.ilike(f"%{name}%"))
-            .options(
-                selectinload(stop_stop_area.areas),
-                selectinload(stop_stop_area.lines),
-            )
-        )
 
-        res = await session.execute(stmt)
+                match_stmt = descendants.names_tsv.match(
-        stops = res.scalars().all()
+                    name, postgresql_regconfig="french"
+                )
+                ranking_stmt = func.ts_rank_cd(
+                    descendants.names_tsv, func.plainto_tsquery("french", name)
+                )
+                stmt = (
+                    select(descendants).filter(match_stmt).order_by(desc(ranking_stmt))
+                )
 
-        return stops
+                res = await session.execute(stmt)
+                stops = res.scalars().all()
+
+                return stops
+
+        return None
 
 
 class Stop(_Stop):
@@ -131,7 +144,7 @@ class StopArea(_Stop):
 
     stops: Mapped[list["Stop"]] = relationship(
         "Stop",
-        secondary=stop_area_stop_association_table,
+        secondary="stop_area_stop_associations",
         back_populates="areas",
         lazy="selectin",
     )
@@ -146,41 +159,43 @@ class StopArea(_Stop):
     async def add_stops(
         cls, stop_area_to_stop_ids: Iterable[tuple[int, int]]
     ) -> int | None:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return None
 
-        stop_area_ids, stop_ids = set(), set()
+            async with session.begin():
-        for stop_area_id, stop_id in stop_area_to_stop_ids:
-            stop_area_ids.add(stop_area_id)
-            stop_ids.add(stop_id)
 
-        stop_areas_res = await session.scalars(
+                stop_area_ids, stop_ids = set(), set()
-            select(StopArea)
+                for stop_area_id, stop_id in stop_area_to_stop_ids:
-            .where(StopArea.id.in_(stop_area_ids))
+                    stop_area_ids.add(stop_area_id)
-            .options(selectinload(StopArea.stops))
+                    stop_ids.add(stop_id)
-        )
-        stop_areas: dict[int, StopArea] = {
-            stop_area.id: stop_area for stop_area in stop_areas_res.all()
-        }
 
-        stop_res = await session.execute(select(Stop).where(Stop.id.in_(stop_ids)))
+                stop_areas_res = await session.scalars(
-        stops: dict[int, Stop] = {stop.id: stop for stop in stop_res.scalars()}
+                    select(StopArea)
+                    .where(StopArea.id.in_(stop_area_ids))
+                    .options(selectinload(StopArea.stops))
+                )
+                stop_areas: dict[int, StopArea] = {
+                    stop_area.id: stop_area for stop_area in stop_areas_res.all()
+                }
 
-        found = 0
+                stop_res = await session.execute(
-        for stop_area_id, stop_id in stop_area_to_stop_ids:
+                    select(Stop).where(Stop.id.in_(stop_ids))
-            if (stop_area := stop_areas.get(stop_area_id)) is not None:
+                )
-                if (stop := stops.get(stop_id)) is not None:
+                stops: dict[int, Stop] = {stop.id: stop for stop in stop_res.scalars()}
-                    stop_area.stops.append(stop)
-                    found += 1
-                else:
-                    print(f"No stop found for {stop_id} id")
-            else:
-                print(f"No stop area found for {stop_area_id}")
 
-        await session.commit()
+                found = 0
+                for stop_area_id, stop_id in stop_area_to_stop_ids:
+                    if (stop_area := stop_areas.get(stop_area_id)) is not None:
+                        if (stop := stops.get(stop_id)) is not None:
+                            stop_area.stops.append(stop)
+                            found += 1
+                        else:
+                            print(f"No stop found for {stop_id} id")
+                    else:
+                        print(f"No stop area found for {stop_area_id}")
 
-        return found
+                return found
+
+        return None
 
 
 class StopShape(Base):
@@ -221,38 +236,40 @@ class ConnectionArea(Base):
     async def add_stops(
         cls, conn_area_to_stop_ids: Iterable[tuple[int, int]]
     ) -> int | None:
-        session = cls.db.session
+        if (session := await cls.db.get_session()) is not None:
-        if session is None:
-            return None
 
-        conn_area_ids, stop_ids = set(), set()
+            async with session.begin():
-        for conn_area_id, stop_id in conn_area_to_stop_ids:
-            conn_area_ids.add(conn_area_id)
-            stop_ids.add(stop_id)
 
-        conn_area_res = await session.execute(
+                conn_area_ids, stop_ids = set(), set()
-            select(ConnectionArea)
+                for conn_area_id, stop_id in conn_area_to_stop_ids:
-            .where(ConnectionArea.id.in_(conn_area_ids))
+                    conn_area_ids.add(conn_area_id)
-            .options(selectinload(ConnectionArea.stops))
+                    stop_ids.add(stop_id)
-        )
-        conn_areas: dict[int, ConnectionArea] = {
-            conn.id: conn for conn in conn_area_res.scalars()
-        }
 
-        stop_res = await session.execute(select(_Stop).where(_Stop.id.in_(stop_ids)))
+                conn_area_res = await session.execute(
-        stops: dict[int, _Stop] = {stop.id: stop for stop in stop_res.scalars()}
+                    select(ConnectionArea)
+                    .where(ConnectionArea.id.in_(conn_area_ids))
+                    .options(selectinload(ConnectionArea.stops))
+                )
+                conn_areas: dict[int, ConnectionArea] = {
+                    conn.id: conn for conn in conn_area_res.scalars()
+                }
 
-        found = 0
+                stop_res = await session.execute(
-        for conn_area_id, stop_id in conn_area_to_stop_ids:
+                    select(Stop).where(Stop.id.in_(stop_ids))
-            if (conn_area := conn_areas.get(conn_area_id)) is not None:
+                )
-                if (stop := stops.get(stop_id)) is not None:
+                stops: dict[int, Stop] = {stop.id: stop for stop in stop_res.scalars()}
-                    conn_area.stops.append(stop)
-                    found += 1
-                else:
-                    print(f"No stop found for {stop_id} id")
-            else:
-                print(f"No connection area found for {conn_area_id}")
 
-        await session.commit()
+                found = 0
+                for conn_area_id, stop_id in conn_area_to_stop_ids:
+                    if (conn_area := conn_areas.get(conn_area_id)) is not None:
+                        if (stop := stops.get(stop_id)) is not None:
+                            conn_area.stops.append(stop)
+                            found += 1
+                        else:
+                            print(f"No stop found for {stop_id} id")
+                    else:
+                        print(f"No connection area found for {conn_area_id}")
 
-        return found
+                return found
+
+        return None

backend/backend/models/user.py

@@ -1,25 +1,27 @@
-from sqlalchemy import Column, ForeignKey, String, Table
+from sqlalchemy import BigInteger, ForeignKey, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from ..db import Base, db
 from .stop import _Stop
 
-user_last_stop_search_stops_associations_table = Table(
+
-    "user_last_stop_search_stops_associations_table",
+class UserLastStopSearchStopAssociations(Base):
-    Base.metadata,
+    id = mapped_column(BigInteger, primary_key=True)
-    Column("user_mxid", ForeignKey("user_last_stop_search_results.user_mxid")),
+    user_mxid = mapped_column(
-    Column("stop_id", ForeignKey("_stops.id")),
+        String, ForeignKey("user_last_stop_search_results.user_mxid")
-)
+    )
+    stop_id = mapped_column(BigInteger, ForeignKey("_stops.id"))
+
+    __tablename__ = "user_last_stop_search_stop_associations"
 
 
 class UserLastStopSearchResults(Base):
-
     db = db
 
-    __tablename__ = "user_last_stop_search_results"
-
     user_mxid = mapped_column(String, primary_key=True)
     request_content = mapped_column(String, nullable=False)
     stops: Mapped[_Stop] = relationship(
-        _Stop, secondary=user_last_stop_search_stops_associations_table
+        _Stop, secondary="user_last_stop_search_stop_associations"
     )
+
+    __tablename__ = "user_last_stop_search_results"

backend/backend/schemas/line.py

@@ -46,7 +46,7 @@ class TransportMode(StrEnum):
 
 
 class Line(BaseModel):
-    id: str
+    id: int
     shortName: str
     name: str
     status: IdfmLineState

backend/backend/settings.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel, BaseSettings, Field, SecretStr
+
+
+class HttpSettings(BaseModel):
+    host: str = "127.0.0.1"
+    port: int = 8080
+    cert: str | None = None
+
+
+class DatabaseSettings(BaseModel):
+    name: str = "carrramba-encore-rate"
+    host: str = "127.0.0.1"
+    port: int = 5432
+    driver: str = "postgresql+psycopg"
+    user: str = "cer"
+    password: SecretStr | None = None
+
+
+class Settings(BaseSettings):
+    app_name: str
+
+    idfm_api_key: SecretStr = Field(..., env="API_KEY")
+    clear_static_data: bool = Field(..., env="CLEAR_STATIC_DATA")
+
+    http: HttpSettings = HttpSettings()
+    db: DatabaseSettings = DatabaseSettings()

backend/config/config.sample.yaml

@@ -0,0 +1,14 @@
+app_name: carrramba-encore-rate
+
+http:
+  host: 0.0.0.0
+  port: 4443
+  cert: ./config/cert.pem
+
+db:
+  name: carrramba-encore-rate
+  host: 127.0.0.1
+  port: 5432
+  driver: postgresql+psycopg
+  user: cer
+  password: cer_password

backend/docker-compose.yml

@@ -13,7 +13,62 @@ services:
         max-size: 10m
         max-file: "3"
     ports:
-      - '127.0.0.1:5438:5432'
+      - "127.0.0.1:5432:5432"
     volumes:
       - ./docker/database/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       - ./docker/database/data:/var/lib/postgresql/data
+
+  jaeger-agent:
+    image: jaegertracing/jaeger-agent:latest
+    command:
+      - "--reporter.grpc.host-port=jaeger-collector:14250"
+    ports:
+      - "127.0.0.1:5775:5775/udp"
+      - "127.0.0.1:6831:6831/udp"
+      - "127.0.0.1:6832:6832/udp"
+      - "127.0.0.1:5778:5778"
+    restart: on-failure
+    depends_on:
+      - jaeger-collector
+
+  jaeger-collector:
+    image: jaegertracing/jaeger-collector:latest
+    command:
+      - "--cassandra.keyspace=jaeger_v1_dc1"
+      - "--cassandra.servers=cassandra"
+      - "--collector.zipkin.host-port=9411"
+      - "--sampling.initial-sampling-probability=.5"
+      - "--sampling.target-samples-per-second=.01"
+    environment:
+      - SAMPLING_CONFIG_TYPE=adaptive
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - "127.0.0.1:4317:4317"
+      - "127.0.0.1:4318:4318"
+      # - "127.0.0.1:9411:9411"
+      # - "127.0.0.1:14250:14250"
+      # - "127.0.0.1:14268:14268"
+      # - "127.0.0.1:14269:14269"
+    restart: on-failure
+    depends_on:
+      - cassandra-schema
+
+  cassandra:
+    image: cassandra:latest
+
+  cassandra-schema:
+    image: jaegertracing/jaeger-cassandra-schema:latest
+    depends_on:
+      - cassandra
+
+  jaeger-query:
+    image: jaegertracing/jaeger-query:latest
+    command:
+      - "--cassandra.keyspace=jaeger_v1_dc1"
+      - "--cassandra.servers=cassandra"
+    ports:
+      - "127.0.0.1:16686:16686"
+      # - "127.0.0.1:16687:16687"
+    restart: on-failure
+    depends_on:
+      - cassandra-schema

backend/main.py

@@ -1,12 +1,23 @@
+#!/usr/bin/env python3
+import logging
 from collections import defaultdict
 from datetime import datetime
 from os import environ, EX_USAGE
 from typing import Sequence
 
+import uvicorn
+from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.sdk.resources import Resource, SERVICE_NAME
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from rich import print
+from yaml import safe_load
 
 from backend.db import db
 from backend.idfm_interface import Destinations as IdfmDestinations, IdfmInterface
@@ -20,46 +31,54 @@ from backend.schemas import (
     StopArea as StopAreaSchema,
     StopShape as StopShapeSchema,
 )
-
+from backend.settings import Settings
-API_KEY = environ.get("API_KEY")
-if API_KEY is None:
-    print('No "API_KEY" environment variable set... abort.')
-    exit(EX_USAGE)
-
-# TODO: Remove postgresql+asyncpg from environ variable
-DB_PATH = "postgresql+asyncpg://cer_user:cer_password@127.0.0.1:5438/cer_db"
 
 
-app = FastAPI()
+CONFIG_PATH = environ.get("CONFIG_PATH", "./config.sample.yaml")
+
+
+def load_settings(path: str) -> Settings:
+    with open(path, "r") as config_file:
+        config = safe_load(config_file)
+
+    return Settings(**config)
+
+
+settings = load_settings(CONFIG_PATH)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await db.connect(settings.db, settings.clear_static_data)
+    if settings.clear_static_data:
+        await idfm_interface.startup()
+
+    yield
+
+    await db.disconnect()
+
+
+app = FastAPI(lifespan=lifespan)
+
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=[
+    allow_origins=["https://localhost:4443", "https://localhost:3000"],
-        "https://localhost:4443",
-        "https://localhost:3000",
-    ],
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
-idfm_interface = IdfmInterface(API_KEY, db)
+app.mount("/widget", StaticFiles(directory="../frontend/", html=True), name="widget")
 
+FastAPIInstrumentor.instrument_app(app)
 
-@app.on_event("startup")
+trace.set_tracer_provider(
-async def startup():
+    TracerProvider(resource=Resource.create({SERVICE_NAME: settings.app_name}))
-    await db.connect(DB_PATH, clear_static_data=True)
+)
-    await idfm_interface.startup()
+trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
-    # await db.connect(DB_PATH, clear_static_data=False)
+tracer = trace.get_tracer(settings.app_name)
-    print("Connected")
 
-
+idfm_interface = IdfmInterface(settings.idfm_api_key.get_secret_value(), db)
-@app.on_event("shutdown")
-async def shutdown():
-    await db.disconnect()
-
-
-STATIC_ROOT = "../frontend/"
-app.mount("/widget", StaticFiles(directory=STATIC_ROOT, html=True), name="widget")
 
 
 def optional_datetime_to_ts(dt: datetime | None) -> int | None:
@@ -67,7 +86,7 @@ def optional_datetime_to_ts(dt: datetime | None) -> int | None:
 
 
 @app.get("/line/{line_id}", response_model=LineSchema)
-async def get_line(line_id: str) -> LineSchema:
+async def get_line(line_id: int) -> LineSchema:
     line: Line | None = await Line.get_by_id(line_id)
 
     if line is None:
@@ -140,15 +159,14 @@ async def get_stop(
             )
         )
 
-    # print(f"{stops = }", flush=True)
     formatted.extend(_format_stop(stop) for stop in stops.values())
 
     return formatted
 
 
 # TODO: Cache response for 30 secs ?
-@app.get("/stop/nextPassages/{stop_id}")
+@app.get("/stop/{stop_id}/nextPassages")
-async def get_next_passages(stop_id: str) -> NextPassagesSchema | None:
+async def get_next_passages(stop_id: int) -> NextPassagesSchema | None:
     res = await idfm_interface.get_next_passages(stop_id)
     if res is None:
         return None
@@ -167,11 +185,12 @@ async def get_next_passages(stop_id: str) -> NextPassagesSchema | None:
 
             # re.match will return None if the given journey.LineRef.value is not valid.
             try:
-                line_id = IdfmInterface.LINE_RE.match(journey.LineRef.value).group(1)
+                line_id_match = IdfmInterface.LINE_RE.match(journey.LineRef.value)
-            except AttributeError as exc:
+                line_id = int(line_id_match.group(1))  # type: ignore
+            except (AttributeError, TypeError, ValueError) as err:
                 raise HTTPException(
                     status_code=404, detail=f'Line "{journey.LineRef.value}" not found'
-                ) from exc
+                ) from err
 
             call = journey.MonitoredCall
 
@@ -215,8 +234,7 @@ async def get_stop_destinations(
     return destinations
 
 
-# TODO: Rename endpoint -> /stop/{stop_id}/shape
+@app.get("/stop/{stop_id}/shape")
-@app.get("/stop_shape/{stop_id}")
 async def get_stop_shape(stop_id: int) -> StopShapeSchema | None:
     connection_area = None
 
@@ -247,3 +265,13 @@ async def get_stop_shape(stop_id: int) -> StopShapeSchema | None:
 
     msg = f"No shape found for stop {stop_id}"
     raise HTTPException(status_code=404, detail=msg)
+
+
+if __name__ == "__main__":
+    http_settings = settings.http
+    uvicorn.run(
+        app,
+        host=http_settings.host,
+        port=http_settings.port,
+        ssl_certfile=http_settings.cert,
+    )

backend/pyproject.toml

@@ -11,13 +11,21 @@ python = "^3.11"
 aiohttp = "^3.8.3"
 rich = "^12.6.0"
 aiofiles = "^22.1.0"
-sqlalchemy = {extras = ["asyncio"], version = "^2.0.1"}
+fastapi = "^0.95.0"
-fastapi = "^0.88.0"
 uvicorn = "^0.20.0"
-asyncpg = "^0.27.0"
 msgspec = "^0.12.0"
 pyshp = "^2.3.1"
 pyproj = "^3.5.0"
+opentelemetry-instrumentation-fastapi = "^0.38b0"
+sqlalchemy-utils = "^0.41.1"
+opentelemetry-instrumentation-logging = "^0.38b0"
+opentelemetry-sdk = "^1.17.0"
+opentelemetry-api = "^1.17.0"
+opentelemetry-exporter-otlp-proto-http = "^1.17.0"
+opentelemetry-instrumentation-sqlalchemy = "^0.38b0"
+sqlalchemy = "^2.0.12"
+psycopg = "^3.1.9"
+pyyaml = "^6.0"
 
 [build-system]
 requires = ["poetry-core"]
@@ -39,8 +47,9 @@ autopep8 = "^2.0.1"
 pyflakes = "^3.0.1"
 yapf = "^0.32.0"
 whatthepatch = "^1.0.4"
-sqlalchemy = {extras = ["mypy"], version = "^2.0.1"}
 mypy = "^1.0.0"
+types-sqlalchemy-utils = "^1.0.1"
+types-pyyaml = "^6.0.12.9"
 
 [tool.mypy]
 plugins = "sqlalchemy.ext.mypy.plugin"

frontend/src/App.tsx

@@ -1,4 +1,4 @@
-import { Component } from 'solid-js';
+import { Component, createSignal } from 'solid-js';
 import { IVisibilityActionRequest, MatrixCapabilities, WidgetApi, WidgetApiToWidgetAction } from 'matrix-widget-api';
 import { HopeProvider } from "@hope-ui/solid";
 
@@ -6,9 +6,10 @@ import { BusinessDataProvider } from './businessData';
 import { AppContextProvider } from './appContext';
 
 import { PassagesDisplay } from './passagesDisplay';
-import { StopsSearchMenu } from './stopsSearchMenu';
+import { StopsSearchMenu } from './stopsSearchMenu/stopsSearchMenu';
 
 import "./App.scss";
+import { onCleanup, onMount } from 'solid-js';
 
 
 function parseFragment() {
@@ -43,6 +44,33 @@ const App: Component = () => {
 		api.transport.reply(ev.detail, {});
 	});
 
+	createSignal({
+		height: window.innerHeight,
+		width: window.innerWidth
+	});
+
+	const onResize = () => {
+		const body = document.body;
+
+		if (window.innerWidth * 9 / 16 < window.innerHeight) {
+			body.style['height'] = 'auto';
+			body.style['width'] = '100vw';
+		}
+		else {
+			body.style['height'] = '100vh';
+			body.style['width'] = 'auto';
+		}
+	};
+
+	onMount(() => {
+		window.addEventListener('resize', onResize);
+		onResize();
+	});
+
+	onCleanup(() => {
+		window.removeEventListener('resize', onResize);
+	})
+
 	return (
 		<BusinessDataProvider>
 			<AppContextProvider>

frontend/src/businessData.tsx

@@ -21,7 +21,7 @@ export interface BusinessDataStore {
 	getStop: (stopId: number) => Stop | undefined;
 	searchStopByName: (name: string) => Promise<Stops>;
 
-	getStopDestinations: (stopId: number) => Promise<StopDestinations>;
+	getStopDestinations: (stopId: number) => Promise<StopDestinations | undefined>;
 	getStopShape: (stopId: number) => Promise<StopShape | undefined>;
 };
 
@@ -44,11 +44,20 @@ export function BusinessDataProvider(props: { children: JSX.Element }) {
 		let line = store.lines[lineId];
 		if (line === undefined) {
 			console.log(`${lineId} not found... fetch it from backend.`);
-			const data = await fetch(`${serverUrl()}/line/${lineId}`, {
+
+			const response = await fetch(`${serverUrl()}/line/${lineId}`, {
 				headers: { 'Content-Type': 'application/json' }
 			});
-			line = await data.json();
+
-			setStore('lines', lineId, line);
+			const json = await response.json();
+
+			if (response.ok) {
+				setStore('lines', lineId, json);
+				line = json;
+			}
+			else {
+				console.warn(`No line found for ${lineId} line id:`, json);
+			}
 		}
 		return line;
 	}
@@ -91,12 +100,19 @@ export function BusinessDataProvider(props: { children: JSX.Element }) {
 	}
 
 	const refreshPassages = async (stopId: number): Promise<void> => {
-		const httpOptions = { headers: { "Content-Type": "application/json" } };
 		console.log(`Fetching data for ${stopId}`);
-		const data = await fetch(`${serverUrl()}/stop/nextPassages/${stopId}`, httpOptions);
+		const httpOptions = { headers: { "Content-Type": "application/json" } };
-		const response = await data.json();
+		const response = await fetch(`${serverUrl()}/stop/${stopId}/nextPassages`, httpOptions);
-		_cleanupPassages(response.passages);
+
-		addPassages(response.passages);
+		const json = await response.json();
+
+		if (response.ok) {
+			_cleanupPassages(json.passages);
+			addPassages(json.passages);
+		}
+		else {
+			console.warn(`No passage found for ${stopId} stop:`, json);
+		}
 	}
 
 	const addPassages = (passages: Passages): void => {
@@ -155,39 +171,58 @@ ${linePassagesDestination.length} here... refresh all them.`);
 	}
 
 	const searchStopByName = async (name: string): Promise<Stops> => {
-		const data = await fetch(`${serverUrl()}/stop/?name=${name}`, {
+		const byIdStops: Stops = {};
+
+		const response = await fetch(`${serverUrl()}/stop/?name=${name}`, {
 			headers: { 'Content-Type': 'application/json' }
 		});
-		const stops = await data.json();
 
-		const byIdStops: Stops = {};
+		const json = await response.json();
-		for (const stop of stops) {
+
-			byIdStops[stop.id] = stop;
+		if (response.ok) {
-			setStore('stops', stop.id, stop);
+			for (const stop of json) {
-			for (const innerStop of stop.stops) {
+				byIdStops[stop.id] = stop;
-				setStore('stops', innerStop.id, innerStop);
+				setStore('stops', stop.id, stop);
+
+				if (stop.stops !== undefined) {
+					for (const innerStop of stop.stops) {
+						setStore('stops', innerStop.id, innerStop);
+					}
+				}
 			}
 		}
+		else {
+			console.warn(`No stop found for '${name}' query:`, json);
+		}
+
 		return byIdStops;
 	}
 
-	const getStopDestinations = async (stopId: number): Promise<StopDestinations> => {
+	const getStopDestinations = async (stopId: number): Promise<StopDestinations | undefined> => {
-		const data = await fetch(`${serverUrl()}/stop/${stopId}/destinations`, {
+		const response = await fetch(`${serverUrl()}/stop/${stopId}/destinations`, {
 			headers: { 'Content-Type': 'application/json' }
 		});
-		const response = await data.json();
+		const destinations = response.ok ? await response.json() : undefined;
-		return response;
+		return destinations;
 	}
 
 	const getStopShape = async (stopId: number): Promise<StopShape | undefined> => {
 		let shape = store.stopShapes[stopId];
 		if (shape === undefined) {
 			console.log(`No shape found for ${stopId} stop... fetch it from backend.`);
-			const data = await fetch(`${serverUrl()}/stop_shape/${stopId}`, {
+			const response = await fetch(`${serverUrl()}/stop/${stopId}/shape`, {
 				headers: { 'Content-Type': 'application/json' }
 			});
-			shape = await data.json();
+
-			setStore('stopShapes', stopId, shape);
+			const json = await response.json();
+
+			if (response.ok) {
+				setStore('stopShapes', stopId, json);
+				shape = json;
+			}
+			else {
+				console.warn(`No shape found for ${stopId} stop:`, json);
+			}
 		}
 		return shape;
 	}

frontend/src/index.scss

@@ -13,10 +13,8 @@
 	src: url(/public/fonts/IDFVoyageur-Medium.otf);
 }
 
-body {
+html, body {
 	aspect-ratio: 16/9;
-	width: 100vw;
-	height: none;
 
 	margin: 0;