diff --git a/docker-compose.yml b/docker-compose.yml index e450741..9d2edbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,22 @@ volumes: postgis-data: - packages: # do not change any detail of this volume + packages: # do not change any detail of this volume driver: local driver_opts: type: none - device: $DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file + device: $DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file + o: bind + tmp_packages: # do not change any detail of this volume + driver: local + driver_opts: + type: none + device: $TMP_DATA_DIR # DATA_DIR is the host directory for the data. It has to be in the environment, e.g. in .env file o: bind # It's important it runs in its own private network, also more secure networks: routing-packager: -version: '3.2' services: postgis: image: kartoza/postgis:12.1 @@ -31,7 +36,8 @@ services: restart: always redis: image: redis:6.2 - container_name: routing-packager-redis + container_name: + routing-packager-redis # mostly needed to define the database hosts env_file: - .docker_env @@ -50,7 +56,8 @@ services: - routing-packager volumes: - packages:/app/data - - $PWD/.docker_env:/app/.env # Worker needs access to .env file + - tmp_packages:/app/tmp_data + - $PWD/.docker_env:/app/.env # Worker needs access to .env file depends_on: - postgis - redis @@ -67,7 +74,8 @@ services: - .docker_env volumes: - packages:/app/data - - $PWD/.docker_env:/app/.env # CLI needs access to .env file + - tmp_packages:/app/tmp_data + - $PWD/.docker_env:/app/.env # CLI needs access to .env file - $PWD/static:/app/static # static file for frontend networks: - routing-packager diff --git a/main.py b/main.py index 3d60c2b..d67a5d7 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +from contextlib import asynccontextmanager import uvicorn as uvicorn from arq import create_pool from arq.connections import RedisSettings @@ -10,11 +11,9 @@ from routing_packager_app.config import SETTINGS from routing_packager_app.api_v1.models import User -app: FastAPI = create_app() - -@app.on_event("startup") -async def startup_event(): +@asynccontextmanager +async def lifespan(app: FastAPI): SQLModel.metadata.create_all(engine, checkfirst=True) app.state.redis_pool = await create_pool(RedisSettings.from_dsn(SETTINGS.REDIS_URL)) User.add_admin_user(next(get_db())) @@ -24,7 +23,11 @@ async def startup_event(): p = SETTINGS.get_data_dir().joinpath(provider.lower()) p.mkdir(exist_ok=True) SETTINGS.get_output_path().mkdir(exist_ok=True) + yield + app.state.redis_pool.shutdown() + +app: FastAPI = create_app(lifespan=lifespan) if __name__ == "__main__": uvicorn.run("main:app", host="0.0.0.0", port=5000, reload=True) diff --git a/routing_packager_app/config.py b/routing_packager_app/config.py index f3a6460..83ac0a2 100644 --- a/routing_packager_app/config.py +++ b/routing_packager_app/config.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import List, Optional -from pydantic import BaseSettings as _BaseSettings +from pydantic_settings import SettingsConfigDict, BaseSettings as _BaseSettings from starlette.datastructures import CommaSeparatedStrings from routing_packager_app.constants import Providers @@ -45,6 +45,8 @@ class BaseSettings(_BaseSettings): SMTP_PASS: str = "" SMTP_SECURE: bool = False + model_config = SettingsConfigDict(extra="ignore") + def get_valhalla_path(self, port: int) -> Path: # pragma: no cover """ Return the path to the OSM Valhalla instances. @@ -69,15 +71,11 @@ def get_data_dir(self) -> Path: class ProdSettings(BaseSettings): - class Config: - case_sensitive = True - env_file = ENV_FILE + model_config = SettingsConfigDict(case_sensitive=True, env_file=ENV_FILE, extra="ignore") class DevSettings(BaseSettings): - class Config: - case_sensitive = True - env_file = ENV_FILE + model_config = SettingsConfigDict(case_sensitive=True, env_file=ENV_FILE, extra="ignore") class TestSettings(BaseSettings): @@ -89,17 +87,18 @@ class TestSettings(BaseSettings): POSTGRES_PASS: str = "admin" DATA_DIR: Path = BASE_DIR.joinpath("tests", "data") + TMP_DATA_DIR: Path = BASE_DIR.joinpath("tests", "tmp_data") ADMIN_EMAIL: str = "admin@example.org" ADMIN_PASS: str = "admin" - - class Config: - case_sensitive = True - env_file = BASE_DIR.joinpath("tests", "env") + model_config = SettingsConfigDict( + case_sensitive=True, env_file=BASE_DIR.joinpath("tests", "env"), extra="ignore" + ) # decide which settings we'll use SETTINGS: Optional[BaseSettings] = None +print("LOADING SETTINGS") env = os.getenv("API_CONFIG", "prod") if env == "prod": # pragma: no cover SETTINGS = ProdSettings() diff --git a/routing_packager_app/worker.py b/routing_packager_app/worker.py index 6b29265..b5fe531 100644 --- a/routing_packager_app/worker.py +++ b/routing_packager_app/worker.py @@ -1,7 +1,7 @@ import json import logging import os -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from arq.connections import RedisSettings @@ -9,8 +9,13 @@ import requests from requests.exceptions import ConnectionError import shutil -from sqlmodel import Session -from starlette.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR, HTTP_301_MOVED_PERMANENTLY +from sqlmodel import Session, select +from starlette.status import ( + HTTP_200_OK, + HTTP_404_NOT_FOUND, + HTTP_500_INTERNAL_SERVER_ERROR, + HTTP_301_MOVED_PERMANENTLY, +) from .api_v1.dependencies import split_bbox from .config import SETTINGS @@ -38,16 +43,30 @@ async def create_package( # Set up the logger where we have access to the user email # and only if there hasn't been one before - user_email = session.query(User).get(user_id).email + statement = select(User).where(User.id == user_id) + results = session.exec(statement).first() + + if results is None: + raise HTTPException( + HTTP_404_NOT_FOUND, + "No user with specified ID found.", + ) + user_email = results.email if not LOGGER.handlers and update is False: handler = AppSmtpHandler(**get_smtp_details([user_email])) handler.setLevel(logging.INFO) LOGGER.addHandler(handler) log_extra = {"user": user_email, "job_id": job_id} - job: Job = session.query(Job).get(job_id) + statement = select(Job).where(Job.id == job_id) + job = session.exec(statement).first() + if job is None: + raise HTTPException( + HTTP_404_NOT_FOUND, + "No job with specified ID found.", + ) job.status = Statuses.COMPRESSING - job.last_started = datetime.utcnow() + job.last_started = datetime.now(timezone.utc) session.commit() succeeded = False @@ -98,7 +117,7 @@ async def create_package( "name": job_name, "description": description, "extent": bbox, - "last_modified": str(datetime.utcnow()), + "last_modified": str(datetime.now(timezone.utc)), } dirname = os.path.dirname(zip_path) fname_sanitized = fname.split(os.extsep, 1)[0] @@ -126,7 +145,7 @@ async def create_package( final_status = Statuses.FAILED # always write the "last_finished" column - job.last_finished = datetime.utcnow() + job.last_finished = datetime.now(timezone.utc) job.status = final_status session.commit()