📝 Add Docs section to the TODO list

Cf. https://bugzilla.mozilla.org/show_bug.cgi?id=1368063

.gitattributes

@@ -0,0 +1,2 @@
+medias/presentation.png filter=lfs diff=lfs merge=lfs -text
+medias/presentation.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,124 @@
+# 🚧 Carrramba! Encore raté! 🚧
+
+Resident of the Ile-de-France,
+
+- Tired of missing your bus/train/metro ?
+- Tired of having to walk to know where your bus/train/metro will stop ?
+- Are you looking to display its next passages in the same way as Ile De France Mobilité ?
+
+**Visit [carrramba.adrien.run](https://carrramba.adrien.run/)**
+
+[![Presentation](medias/presentation.png)](medias/presentation.mp4)
+
+# Technical stack
+
+The following figure list the building blocks (docker images) and their interactions to provide the service:
+
+```plantuml
+@startuml
+!include <C4/C4_Container>
+
+!define ICONURL https://raw.githubusercontent.com/tupadr3/plantuml-icon-font-sprites/v2.4.0
+!includeurl ICONURL/common.puml
+
+!includeurl ICONURL/font-awesome-5/users.puml
+!includeurl ICONURL/devicons2/redis.puml
+!includeurl ICONURL/devicons2/sqlalchemy.puml
+!includeurl ICONURL/devicons2/postgresql.puml
+
+title Carrramba-encore-rate - Back-end
+
+LAYOUT_WITH_LEGEND()
+
+Person(user, "User", "The service user", $sprite="users")
+
+System_Boundary(cb1, "carrramba-encore-rate") {
+  Container(app, "SPA", "SolidJS", "The graphical interface used by users to consume services provided")
+  Container(api, "carrramba-encore-rate-api", "FastAPI / Pydantic 2 / SQLAlchemy 2", "Provides the functionalities serving API endpoints", $sprite="sqlalchemy")
+  ContainerDb(db, "Postgres", "PostgreSQL database", "Stores stops/stop areas/lines/shapes and associations information.", $sprite="postgresql")
+  ContainerDb(cache, "In-Memory cache", "Redis", "Store previously computed results (stop, line, destination, next passages, shapes).", $sprite="redis")
+
+  Container(db_updater, "db-updater", "Sync the service static data with the IDFM ones")
+}
+
+System_Boundary(idfm, "IDFM") {
+  Container_Ext(static_, "Static data")
+  Container_Ext(dynamic_, "Dynamic data")
+}
+
+Rel(user, app, "Uses")
+
+Rel(app, api, "Uses", "JSON/HTTPS")
+
+Rel_R(api, db, "Reads from", "sync, PSQL")
+Rel_L(api, cache, "Reads from and writes to", "sync, REdis Serialization Protocol")
+Rel(api, dynamic_, "Get next passages", "JSON/HTTPS")
+
+Rel_L(db_updater, db, "Writes to", "sync, PSQL")
+Rel(db_updater, static_, "Get stops, lines andshapes")
+@enduml
+```
+
+## Back-end
+
+Conventional but efficient:
+
+- [FastAPI](https://fastapi.tiangolo.com/): _FastAPI is a modern, fast (high-performance), web framework for building
+   APIs with Python 3.8+ based on standard Python type hints._
+- [Pydantic 2](https://docs.pydantic.dev/latest/): _Pydantic is the most widely used data validation library for
+   Python._
+- [Sqlalchemy 2](https://www.sqlalchemy.org/): _SQLAlchemy is the Python SQL toolkit and Object Relational Mapper that
+   gives application developers the full power and flexibility of SQL._
+
+The [Msgspec](https://github.com/jcrist/msgspec) library is used to serialize JSON objects collected from IDFM API
+endpoints, this library being faster than Pydantic.
+
+## Front-end
+
+The component is developed using the [SolidJS](https://www.solidjs.com/) library. It brings the following interesting specificities:
+
+- Share with React the same programmatic structures and support for component.
+- Fine-grained reactivity architecture: no virtual DOM used to update the components, the browser DOM is directly
+  updated by SolidJS.
+
+The front-end tries to be as close as possible to the design defined by the IDFM for the displays deployed by the
+transport operators in Ile-de-France. These specifications are public and available here:
+[PRIM-IDFM](https://prim.iledefrance-mobilites.fr/en/chartes-et-prescriptions).
+
+
+# TODO
+
+## Features
+
+- [ ] Integration with [Matrix.org](https://matrix.org/ecosystem/integrations/) ecosystem: make the app able to send message to a room when a bus/train/metro will
+      arriving in X minutes.
+- [ ] Add the capability for the users to pin his/her favorite stops.
+- [ ] Add the address to the stop location.
+
+## Docs
+
+- [ ] Describe how to build the front-end and back-end docker images.
+- [ ] Describe how to deploy them using `docker compose`.
+- [ ] Add back-end API description ([openAPI](https://www.openapis.org/)) + generate documentation
+      ([mkdoc](https://www.mkdocs.org/), [Redoc](https://github.com/Redocly/redoc)). The best would be to build and
+      deploy it (as docker images) using CI/CD... need to test [Agola](https://agola.io/), [Jaypore
+      CI](https://www.jayporeci.in/) (pipelines configured in Python),
+      [Woodpecker](https://woodpecker-ci.org/docs/intro) or [Drone](https://www.drone.io/) before.
+
+## Front-end
+
+- [ ] Add unit tests.
+- [ ] Make the StopNameInput component liquid.
+- [ ] Liquid to responsive Design.
+
+## Back-end
+
+- [ ] Add unit tests.
+- [ ] Use [alembic](https://alembic.sqlalchemy.org/en/latest/) to manage the future updates of the database schemas.
+- [ ] Add the capability to reload the application configuration on configuration file update (e.g.: credential update by the vault).
+- [ ] Rework how the database is updated with the IDFM data: For now the database is cleaned before refilling. It could
+      be useful to avoid to empty the database and only apply deltas (update/add/remove rows).
+- [ ] Could be nice to compare FastAPI with [Litestar](https://litestar.dev/).
+
+<!--  LocalWords:  specificities
+ -->

backend/Dockerfile.api_server

@@ -9,7 +9,7 @@ ENV POETRY_NO_INTERACTION=1 \
 
 WORKDIR /app
 
-COPY ./pyproject.toml /app
+COPY pyproject.toml /app
 
 RUN poetry install --only=main --no-root && \
 	rm -rf ${POETRY_CACHE_DIR}
@@ -29,10 +29,8 @@ env VIRTUAL_ENV=/app/.venv \
 
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-COPY backend /app/backend
-COPY dependencies.py /app
-COPY config.sample.yaml /app
-COPY routers/ /app/routers
-COPY main.py /app
+COPY api /app/api
+COPY config.sample.yaml .
+COPY api_server.py .
 
-CMD ["python", "./main.py"]
+CMD ["./api_server.py"]

backend/Dockerfile.db_updater

@@ -34,10 +34,8 @@ env VIRTUAL_ENV=/app/.venv \
 
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-COPY backend /app/backend
-COPY dependencies.py /app
-COPY config.sample.yaml /app
-COPY config.local.yaml /app
-COPY db_updater /app/db_updater
+COPY api /app/api
+COPY config.sample.yaml .
+COPY db_updater.py .
 
-CMD ["python", "-m", "db_updater.fill_db"]
+CMD ["./db_updater.py"]

backend/README.md

@@ -0,0 +1,9 @@
+
+
+# Architecture
+
+
+
+The following schema shows the components used to satisfy the `api` service:
+
+![API](./docs/medias/hubble-ui_api.png)

backend/api/db/db.py

@@ -61,9 +61,6 @@ class Database:
             while not ret:
                 try:
                     async with self._async_engine.begin() as session:
-                        await session.execute(
-                            text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-                        )
                         if clear_static_data:
                             await session.run_sync(Base.metadata.drop_all)
                             await session.run_sync(Base.metadata.create_all)

backend/api/dependencies.py

@@ -4,9 +4,9 @@ from fastapi_cache.backends.redis import RedisBackend
 from redis import asyncio as aioredis
 from yaml import safe_load
 
-from backend.db import db
-from backend.idfm_interface.idfm_interface import IdfmInterface
-from backend.settings import CacheSettings, Settings
+from .db import db
+from .idfm_interface.idfm_interface import IdfmInterface
+from .settings import CacheSettings, Settings
 
 
 CONFIG_PATH = environ.get("CONFIG_PATH", "./config.sample.yaml")

backend/api/idfm_interface/idfm_interface.py

@@ -7,9 +7,9 @@ from aiohttp import ClientSession
 from msgspec import ValidationError
 from msgspec.json import Decoder
 
+from .idfm_types import Destinations as IdfmDestinations, IdfmResponse, IdfmState
 from ..db import Database
 from ..models import Line, Stop, StopArea
-from .idfm_types import Destinations as IdfmDestinations, IdfmResponse, IdfmState
 
 
 class IdfmInterface:

backend/api/idfm_interface/idfm_types.py

@@ -141,6 +141,8 @@ class StopAreaStopAssociationFields(Struct, kw_only=True):
     artid: str | None = None
     arrversion: str
     zdcid: str
+    pdeid: str | None = None
+    pdeversion: int | None = None
     version: int
     zdaid: str
     zdaversion: str
@@ -205,7 +207,7 @@ class Line(Struct):
 
 Lines = dict[str, Line]
 
-Destinations = dict[str, set[str]]
+Destinations = dict[int, set[str]]
 
 
 # TODO: Set structs frozen

backend/api/routers/line.py

@@ -1,8 +1,8 @@
 from fastapi import APIRouter, HTTPException
 from fastapi_cache.decorator import cache
 
-from backend.models import Line
-from backend.schemas import Line as LineSchema, TransportMode
+from ..models import Line
+from ..schemas import Line as LineSchema, TransportMode
 
 
 router = APIRouter(prefix="/line", tags=["line"])

backend/api/routers/stop.py

@@ -5,16 +5,16 @@ from typing import Sequence
 from fastapi import APIRouter, HTTPException
 from fastapi_cache.decorator import cache
 
-from backend.idfm_interface import Destinations as IdfmDestinations, TrainStatus
-from backend.models import Stop, StopArea, StopShape
-from backend.schemas import (
+from ..idfm_interface import Destinations as IdfmDestinations, TrainStatus
+from ..models import Stop, StopArea, StopShape
+from ..schemas import (
     NextPassage as NextPassageSchema,
     NextPassages as NextPassagesSchema,
     Stop as StopSchema,
     StopArea as StopAreaSchema,
     StopShape as StopShapeSchema,
 )
-from dependencies import idfm_interface
+from ..dependencies import idfm_interface
 
 
 router = APIRouter(prefix="/stop", tags=["stop"])

backend/api/settings.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from typing import Annotated
+
+from pydantic import BaseModel, SecretStr
+from pydantic.functional_validators import model_validator
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
+
+
+class HttpSettings(BaseModel):
+    host: str = "127.0.0.1"
+    port: int = 8080
+    cert: str | None = None
+
+
+class DatabaseSettings(BaseModel):
+    name: str
+    host: str
+    port: int
+    driver: str = "postgresql+psycopg"
+    user: str
+    password: Annotated[SecretStr, check_user_password]
+
+
+class CacheSettings(BaseModel):
+    enable: bool = False
+    host: str = "127.0.0.1"
+    port: int = 6379
+    user: str | None = None
+    password: Annotated[SecretStr | None, check_user_password] = None
+
+
+@model_validator(mode="after")
+def check_user_password(self) -> DatabaseSettings | CacheSettings:
+    if self.user is not None and self.password is None:
+        raise ValueError("user is set, password shall be set too.")
+
+    if self.password is not None and self.user is None:
+        raise ValueError("password is set, user shall be set too.")
+
+    return self
+
+
+class TracingSettings(BaseModel):
+    enable: bool = False
+
+
+class Settings(BaseSettings):
+    app_name: str
+
+    idfm_api_key: SecretStr
+    clear_static_data: bool
+
+    http: HttpSettings
+    db: DatabaseSettings
+    cache: CacheSettings
+    tracing: TracingSettings
+
+    model_config = SettingsConfigDict(env_prefix="CER__", env_nested_delimiter="__")
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return env_settings, init_settings, file_secret_settings

backend/api_server.py

@@ -12,9 +12,9 @@ from opentelemetry.sdk.resources import Resource, SERVICE_NAME
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
-from backend.db import db
-from dependencies import idfm_interface, redis_backend, settings
-from routers import line, stop
+from api.db import db
+from api.dependencies import idfm_interface, redis_backend, settings
+from api.routers import line, stop
 
 
 @asynccontextmanager
@@ -34,13 +34,12 @@ app = FastAPI(lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["https://localhost:4443", "https://localhost:3000"],
+    allow_origins=["http://carrramba.adrien.run", "https://carrramba.adrien.run"],
     allow_credentials=True,
-    allow_methods=["*"],
+    allow_methods=["OPTIONS", "GET"],
     allow_headers=["*"],
 )
 
-
 # The cache-control header entry is not managed properly by fastapi-cache:
 # For now, a request with a cache-control set to no-cache
 # is interpreted as disabling the use of the server cache.

backend/backend/settings.py

@@ -1,59 +0,0 @@
-from typing import Any
-
-from pydantic import BaseModel, BaseSettings, Field, root_validator, SecretStr
-
-
-class HttpSettings(BaseModel):
-    host: str = "127.0.0.1"
-    port: int = 8080
-    cert: str | None = None
-
-
-def check_user_password(cls, values: dict[str, Any]) -> dict[str, Any]:
-    user = values.get("user")
-    password = values.get("password")
-
-    if user is not None and password is None:
-        raise ValueError("user is set, password shall be set too.")
-
-    if password is not None and user is None:
-        raise ValueError("password is set, user shall be set too.")
-
-    return values
-
-
-class DatabaseSettings(BaseModel):
-    name: str = "carrramba-encore-rate"
-    host: str = "127.0.0.1"
-    port: int = 5432
-    driver: str = "postgresql+psycopg"
-    user: str | None = None
-    password: SecretStr | None = None
-
-    _user_password_validation = root_validator(allow_reuse=True)(check_user_password)
-
-
-class CacheSettings(BaseModel):
-    enable: bool = False
-    host: str = "127.0.0.1"
-    port: int = 6379
-    user: str | None = None
-    password: SecretStr | None = None
-
-    _user_password_validation = root_validator(allow_reuse=True)(check_user_password)
-
-
-class TracingSettings(BaseModel):
-    enable: bool = False
-
-
-class Settings(BaseSettings):
-    app_name: str
-
-    idfm_api_key: SecretStr = Field(..., env="IDFM_API_KEY")
-    clear_static_data: bool = Field(False, env="CLEAR_STATIC_DATA")
-
-    http: HttpSettings = HttpSettings()
-    db: DatabaseSettings = DatabaseSettings()
-    cache: CacheSettings = CacheSettings()
-    tracing: TracingSettings = TracingSettings()

backend/config.local.yaml

@@ -2,7 +2,7 @@ app_name: carrramba-encore-rate
 clear_static_data: false
 
 http:
-  host: 0.0.0.0
+  host: 127.0.0.1
   port: 8080
   cert: ./config/cert.pem
 

backend/config.sample.yaml

@@ -10,9 +10,7 @@ db:
   name: carrramba-encore-rate
   host: postgres
   port: 5432
-  driver: postgresql+psycopg
   user: cer
-  password: cer_password
 
 cache:
   enable: true

backend/db_updater.py

@@ -16,9 +16,9 @@ from shapefile import Reader as ShapeFileReader, ShapeRecord  # type: ignore
 from tqdm import tqdm
 from yaml import safe_load
 
-from backend.db import Base, db, Database
-from backend.models import ConnectionArea, Line, LinePicto, Stop, StopArea, StopShape
-from backend.idfm_interface.idfm_types import (
+from api.db import Base, db, Database
+from api.models import ConnectionArea, Line, LinePicto, Stop, StopArea, StopShape
+from api.idfm_interface.idfm_types import (
     ConnectionArea as IdfmConnectionArea,
     IdfmLineState,
     Line as IdfmLine,
@@ -31,8 +31,8 @@ from backend.idfm_interface.idfm_types import (
     StopLineAsso as IdfmStopLineAsso,
     TransportMode,
 )
-from backend.idfm_interface.ratp_types import Picto as RatpPicto
-from backend.settings import Settings
+from api.idfm_interface.ratp_types import Picto as RatpPicto
+from api.settings import Settings
 
 
 CONFIG_PATH = environ.get("CONFIG_PATH", "./config.sample.yaml")

backend/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 python = "^3.11"
 aiohttp = "^3.8.3"
 aiofiles = "^22.1.0"
-fastapi = "^0.95.0"
+fastapi = "^0.103.0"
 uvicorn = "^0.20.0"
 msgspec = "^0.12.0"
 opentelemetry-instrumentation-fastapi = "^0.38b0"
@@ -23,11 +23,12 @@ sqlalchemy = "^2.0.12"
 psycopg = "^3.1.9"
 pyyaml = "^6.0"
 fastapi-cache2 = {extras = ["redis"], version = "^0.2.1"}
+pydantic-settings = "^2.0.3"
 
 [tool.poetry.group.db_updater.dependencies]
 aiofiles = "^22.1.0"
 aiohttp = "^3.8.3"
-fastapi = "^0.95.0"
+fastapi = "^0.103.0"
 msgspec = "^0.12.0"
 opentelemetry-instrumentation-fastapi = "^0.38b0"
 opentelemetry-instrumentation-sqlalchemy = "^0.38b0"
@@ -41,11 +42,14 @@ pyyaml = "^6.0"
 sqlalchemy = "^2.0.12"
 sqlalchemy-utils = "^0.41.1"
 tqdm = "^4.65.0"
+pydantic-settings = "^2.0.3"
 
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
+
+
 [tool.poetry.group.dev.dependencies]
 pylsp-mypy = "^0.6.2"
 mccabe = "^0.7.0"

frontend/src/businessData.tsx

@@ -28,8 +28,7 @@ export interface BusinessDataStore {
 export const BusinessDataContext = createContext<BusinessDataStore>();
 
 export function BusinessDataProvider(props: { children: JSX.Element }) {
-
-	const [serverUrl] = createSignal<string>("https://localhost:4443");
+	const [serverUrl] = createSignal<string>("https://carrramba.adrien.run/api");
 
 	type Store = {
 		lines: Lines;

frontend/src/utils.tsx

@@ -26,7 +26,7 @@ export const TransportModeWeights: Record<string, number> = {
 export function getTransportModeSrc(mode: string, color: boolean = true): string | undefined {
 	let ret = undefined;
 	if (validTransportModes.includes(mode)) {
-		return `/carrramba-encore-rate/public/symbole_${mode}_${color ? "" : "support_fonce_"}RVB.svg`;
+		return `/symbole_${mode}_${color ? "" : "support_fonce_"}RVB.svg`;
 	}
 	return ret;
 }