From 94b75b672833b992f8e5dbed400ae3af0665e31c Mon Sep 17 00:00:00 2001 From: Laurens Weijs Date: Thu, 6 Jun 2024 11:48:10 +0200 Subject: [PATCH] Add storage system for system card --- poetry.lock | 64 ++++++++-------- pyproject.toml | 2 + tad/core/config.py | 2 +- tad/migrations/env.py | 5 +- .../versions/006c480a1920_a_message.py | 1 - .../versions/eb2eed884ae9_a_message.py | 1 - tad/models/system_card.py | 6 ++ tad/services/storage.py | 47 ++++++++++++ tad/services/tasks.py | 10 ++- tests/core/test_exceptions.py | 2 +- tests/services/test_storage.py | 75 +++++++++++++++++++ tests/services/test_system_cards.py | 23 ++++++ 12 files changed, 197 insertions(+), 41 deletions(-) create mode 100644 tad/models/system_card.py create mode 100644 tad/services/storage.py create mode 100644 tests/services/test_storage.py create mode 100644 tests/services/test_system_cards.py diff --git a/poetry.lock b/poetry.lock index 0bb208ea..2ba360ee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "alembic" @@ -750,13 +750,13 @@ files = [ [[package]] name = "nodeenv" -version = "1.9.0" +version = "1.9.1" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ - {file = "nodeenv-1.9.0-py2.py3-none-any.whl", hash = "sha256:508ecec98f9f3330b636d4448c0f1a56fc68017c68f1e7857ebc52acf0eb879a"}, - {file = "nodeenv-1.9.0.tar.gz", hash = "sha256:07f144e90dae547bf0d4ee8da0ee42664a42a04e02ed68e06324348dafe4bdb1"}, + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] [[package]] @@ -1087,13 +1087,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pydantic-settings" -version = "2.3.0" +version = "2.3.1" description = "Settings management using Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_settings-2.3.0-py3-none-any.whl", hash = "sha256:26eeed27370a9c5e3f64e4a7d6602573cbedf05ed940f1d5b11c3f178427af7a"}, - {file = "pydantic_settings-2.3.0.tar.gz", hash = "sha256:78db28855a71503cfe47f39500a1dece523c640afd5280edb5c5c9c9cfa534c9"}, + {file = "pydantic_settings-2.3.1-py3-none-any.whl", hash = "sha256:acb2c213140dfff9669f4fe9f8180d43914f51626db28ab2db7308a576cce51a"}, + {file = "pydantic_settings-2.3.1.tar.gz", hash = "sha256:e34bbd649803a6bb3e2f0f58fb0edff1f0c7f556849fda106cc21bcce12c30ab"}, ] [package.dependencies] @@ -1137,13 +1137,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyright" -version = "1.1.365" +version = "1.1.366" description = "Command line wrapper for pyright" optional = false python-versions = ">=3.7" files = [ - {file = "pyright-1.1.365-py3-none-any.whl", hash = "sha256:194d767a039f9034376b7ec8423841880ac6efdd061f3e283b4ad9fcd484a659"}, - {file = "pyright-1.1.365.tar.gz", hash = "sha256:d7e69000939aed4bf823707086c30c84c005bdd39fac2dfb370f0e5be16c2ef2"}, + {file = "pyright-1.1.366-py3-none-any.whl", hash = "sha256:c09e73ccc894976bcd6d6a5784aa84d724dbd9ceb7b873b39d475ca61c2de071"}, + {file = "pyright-1.1.366.tar.gz", hash = "sha256:10e4d60be411f6d960cd39b0b58bf2ff76f2c83b9aeb102ffa9d9fda2e1303cb"}, ] [package.dependencies] @@ -1155,13 +1155,13 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "8.2.1" +version = "8.2.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, - {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] [package.dependencies] @@ -1356,28 +1356,28 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.4.7" +version = "0.4.8" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.4.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e089371c67892a73b6bb1525608e89a2aca1b77b5440acf7a71dda5dac958f9e"}, - {file = "ruff-0.4.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:10f973d521d910e5f9c72ab27e409e839089f955be8a4c8826601a6323a89753"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59c3d110970001dfa494bcd95478e62286c751126dfb15c3c46e7915fc49694f"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa9773c6c00f4958f73b317bc0fd125295110c3776089f6ef318f4b775f0abe4"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07fc80bbb61e42b3b23b10fda6a2a0f5a067f810180a3760c5ef1b456c21b9db"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:fa4dafe3fe66d90e2e2b63fa1591dd6e3f090ca2128daa0be33db894e6c18648"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7c0083febdec17571455903b184a10026603a1de078428ba155e7ce9358c5f6"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad1b20e66a44057c326168437d680a2166c177c939346b19c0d6b08a62a37589"}, - {file = "ruff-0.4.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbf5d818553add7511c38b05532d94a407f499d1a76ebb0cad0374e32bc67202"}, - {file = "ruff-0.4.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:50e9651578b629baec3d1513b2534de0ac7ed7753e1382272b8d609997e27e83"}, - {file = "ruff-0.4.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8874a9df7766cb956b218a0a239e0a5d23d9e843e4da1e113ae1d27ee420877a"}, - {file = "ruff-0.4.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b9de9a6e49f7d529decd09381c0860c3f82fa0b0ea00ea78409b785d2308a567"}, - {file = "ruff-0.4.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:13a1768b0691619822ae6d446132dbdfd568b700ecd3652b20d4e8bc1e498f78"}, - {file = "ruff-0.4.7-py3-none-win32.whl", hash = "sha256:769e5a51df61e07e887b81e6f039e7ed3573316ab7dd9f635c5afaa310e4030e"}, - {file = "ruff-0.4.7-py3-none-win_amd64.whl", hash = "sha256:9e3ab684ad403a9ed1226894c32c3ab9c2e0718440f6f50c7c5829932bc9e054"}, - {file = "ruff-0.4.7-py3-none-win_arm64.whl", hash = "sha256:10f2204b9a613988e3484194c2c9e96a22079206b22b787605c255f130db5ed7"}, - {file = "ruff-0.4.7.tar.gz", hash = "sha256:2331d2b051dc77a289a653fcc6a42cce357087c5975738157cd966590b18b5e1"}, + {file = "ruff-0.4.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066"}, + {file = "ruff-0.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa"}, + {file = "ruff-0.4.8-py3-none-win32.whl", hash = "sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9"}, + {file = "ruff-0.4.8-py3-none-win_amd64.whl", hash = "sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d"}, + {file = "ruff-0.4.8-py3-none-win_arm64.whl", hash = "sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780"}, + {file = "ruff-0.4.8.tar.gz", hash = "sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268"}, ] [[package]] @@ -1962,4 +1962,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "7fe0e4b5d6663a2388e78f1b8b8ce587365f14802424dc5e55e75ce16bac26c8" +content-hash = "b7b2cd05bc3f91a89a4f12e06695e24be11b92c61c271d83b2e9890c855d184b" diff --git a/pyproject.toml b/pyproject.toml index 69b37522..a587c4f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ jinja2 = "^3.1.4" pydantic-settings = "^2.2.1" psycopg2-binary = "^2.9.9" uvicorn = {extras = ["standard"], version = "^0.30.1"} +pyyaml = "^6.0.1" [tool.poetry.group.test.dependencies] @@ -66,6 +67,7 @@ line-ending = "lf" select = ["I", "SIM", "B", "UP", "F", "E", "S", "C90", "DTZ", "LOG", "PIE", "PT", "ERA", "W", "C", "TRY", "RUF"] fixable = ["ALL"] task-tags = ["TODO"] +ignore = ["TRY003"] [tool.ruff.lint.per-file-ignores] "tests/**.py" = ["S101"] diff --git a/tad/core/config.py b/tad/core/config.py index a783aa6e..a1bec7ae 100644 --- a/tad/core/config.py +++ b/tad/core/config.py @@ -85,7 +85,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str: @model_validator(mode="after") def _enforce_database_rules(self: SelfSettings) -> SelfSettings: if self.ENVIRONMENT != "local" and self.APP_DATABASE_SCHEME == "sqlite": - raise SettingsError("SQLite is not supported in production") # noqa: TRY003 + raise SettingsError("SQLite is not supported in production") return self diff --git a/tad/migrations/env.py b/tad/migrations/env.py index f8e8ffcd..258d0520 100644 --- a/tad/migrations/env.py +++ b/tad/migrations/env.py @@ -1,12 +1,11 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool from sqlmodel import SQLModel from tad.models import * # noqa -from alembic import context - config = context.config if config.config_file_name is not None: @@ -58,7 +57,7 @@ def run_migrations_online(): """ configuration = config.get_section(config.config_ini_section) if configuration is None: - raise Exception("Failed to get configuration section") # noqa: TRY003, TRY002 + raise Exception("Failed to get configuration section") # noqa: TRY002 configuration["sqlalchemy.url"] = get_url() connectable = engine_from_config( configuration, diff --git a/tad/migrations/versions/006c480a1920_a_message.py b/tad/migrations/versions/006c480a1920_a_message.py index 8d62a9c2..fa83e759 100644 --- a/tad/migrations/versions/006c480a1920_a_message.py +++ b/tad/migrations/versions/006c480a1920_a_message.py @@ -10,7 +10,6 @@ import sqlalchemy as sa import sqlmodel.sql.sqltypes - from alembic import op # revision identifiers, used by Alembic. diff --git a/tad/migrations/versions/eb2eed884ae9_a_message.py b/tad/migrations/versions/eb2eed884ae9_a_message.py index ce9f87d8..4a861421 100644 --- a/tad/migrations/versions/eb2eed884ae9_a_message.py +++ b/tad/migrations/versions/eb2eed884ae9_a_message.py @@ -10,7 +10,6 @@ import sqlalchemy as sa import sqlmodel.sql.sqltypes - from alembic import op # revision identifiers, used by Alembic. diff --git a/tad/models/system_card.py b/tad/models/system_card.py new file mode 100644 index 00000000..6ab8cb02 --- /dev/null +++ b/tad/models/system_card.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel +from pydantic import Field as PydanticField # type: ignore + + +class SystemCard(BaseModel): + title: str = PydanticField(default=None) diff --git a/tad/services/storage.py b/tad/services/storage.py new file mode 100644 index 00000000..f62de852 --- /dev/null +++ b/tad/services/storage.py @@ -0,0 +1,47 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any + +from yaml import dump + + +class Writer(ABC): + @abstractmethod + def write(self, data: dict[str, Any]) -> None: + """This is an abstract method to write with the writer""" + + @abstractmethod + def close(self) -> None: + """This is an abstract method to close the writer""" + + +class WriterFactory: + @staticmethod + def get_writer(writer_type: str = "file", **kwargs: str) -> Writer: + match writer_type: + case "file": + if not all(k in kwargs for k in ("location", "filename")): + raise KeyError("The `location` or `filename` variables are not provided as input for get_writer()") + return FileSystemWriteService(location=str(kwargs["location"]), filename=str(kwargs["filename"])) + case _: + raise ValueError(f"Unknown writer type: {writer_type}") + + +class FileSystemWriteService(Writer): + def __init__(self, location: str = "./tests/data", filename: str = "system_card.yaml") -> None: + self.location = location + if not filename.endswith(".yaml"): + raise ValueError(f"Filename {filename} must end with .yaml instead of .{filename.split('.')[-1]}") + self.filename = filename + + def write(self, data: dict[str, Any]) -> None: + if not Path(self.location).exists(): + Path(self.location).mkdir() + with open(Path(self.location) / self.filename, "w") as f: + dump(data, f, default_flow_style=False, sort_keys=False) + + def close(self): + """ + This method is empty because with the `with` statement in the writer, Python will already close the writer + after usage. + """ diff --git a/tad/services/tasks.py b/tad/services/tasks.py index b4ffd379..363b821d 100644 --- a/tad/services/tasks.py +++ b/tad/services/tasks.py @@ -4,10 +4,12 @@ from fastapi import Depends +from tad.models.system_card import SystemCard from tad.models.task import Task from tad.models.user import User from tad.repositories.tasks import TasksRepository from tad.services.statuses import StatusesService +from tad.services.storage import WriterFactory logger = logging.getLogger(__name__) @@ -20,6 +22,10 @@ def __init__( ): self.repository = repository self.statuses_service = statuses_service + self.storage_writer = WriterFactory.get_writer( + writer_type="file", location="./output", filename="system_card.yaml" + ) + self.system_card = SystemCard() def get_tasks(self, status_id: int) -> Sequence[Task]: return self.repository.find_by_status_id(status_id) @@ -43,8 +49,8 @@ def move_task( task = self.repository.find_by_id(task_id) if status.name == "done": - # TODO implement logic for done - logging.warning("Task is done, we need to update a system card") + self.system_card.title = task.title + self.storage_writer.write(self.system_card.model_dump()) # assign the task to the current user if status.name == "in_progress": diff --git a/tests/core/test_exceptions.py b/tests/core/test_exceptions.py index 62891784..a2801466 100644 --- a/tests/core/test_exceptions.py +++ b/tests/core/test_exceptions.py @@ -4,6 +4,6 @@ def test_environment_settings(): with pytest.raises(SettingsError) as exc_info: - raise SettingsError("Wrong settings") # noqa: TRY003 + raise SettingsError("Wrong settings") assert exc_info.value.message == "Wrong settings" diff --git a/tests/services/test_storage.py b/tests/services/test_storage.py new file mode 100644 index 00000000..d29aab8f --- /dev/null +++ b/tests/services/test_storage.py @@ -0,0 +1,75 @@ +from pathlib import Path + +import pytest +from tad.models.system_card import SystemCard +from tad.services.storage import WriterFactory +from yaml import safe_load + + +@pytest.fixture() +def setup_and_teardown(tmp_path: Path) -> tuple[str, str]: + filename = "test.yaml" + return filename, str(tmp_path.absolute()) + + +def test_file_system_writer_empty_yaml(setup_and_teardown: tuple[str, str]) -> None: + filename, location = setup_and_teardown + + storage_writer = WriterFactory.get_writer(writer_type="file", location=location, filename=filename) + storage_writer.write({}) + + assert Path.is_file(Path(location) / filename), True + + +def test_file_system_writer_no_location_variable(setup_and_teardown: tuple[str, str]) -> None: + filename, _ = setup_and_teardown + with pytest.raises( + KeyError, match="The `location` or `filename` variables are not provided as input for get_writer()" + ): + WriterFactory.get_writer(writer_type="file", filename=filename) + + +def test_file_system_writer_no_filename_variable(setup_and_teardown: tuple[str, str]) -> None: + _, location = setup_and_teardown + with pytest.raises( + KeyError, match="The `location` or `filename` variables are not provided as input for get_writer()" + ): + WriterFactory.get_writer(writer_type="file", location=location) + + +def test_file_system_writer_yaml_with_content(setup_and_teardown: tuple[str, str]) -> None: + filename, location = setup_and_teardown + data = {"test": "test"} + storage_writer = WriterFactory.get_writer(writer_type="file", location=location, filename=filename) + storage_writer.write(data) + + with open(Path(location) / filename) as f: + assert safe_load(f) == data, True + + +def test_file_system_writer_with_system_card(setup_and_teardown: tuple[str, str]) -> None: + filename, location = setup_and_teardown + data = SystemCard() + data.title = "test" + data_dict = data.model_dump() + + storage_writer = WriterFactory.get_writer(writer_type="file", location=location, filename=filename) + storage_writer.write(data_dict) + + with open(Path(location) / filename) as f: + assert safe_load(f) == data_dict, True + + +def test_abstract_writer_non_yaml_filename(setup_and_teardown: tuple[str, str]) -> None: + _, location = setup_and_teardown + filename = "test.csv" + with pytest.raises( + ValueError, match=f"Filename {filename} must end with .yaml instead of .{filename.split('.')[-1]}" + ): + WriterFactory.get_writer(writer_type="file", location=location, filename=filename) + + +def test_not_valid_writer_type() -> None: + writer_type = "kafka" + with pytest.raises(ValueError, match=f"Unknown writer type: {writer_type}"): + WriterFactory.get_writer(writer_type=writer_type) diff --git a/tests/services/test_system_cards.py b/tests/services/test_system_cards.py new file mode 100644 index 00000000..dd06985c --- /dev/null +++ b/tests/services/test_system_cards.py @@ -0,0 +1,23 @@ +import pytest +from tad.models.system_card import SystemCard + + +@pytest.fixture() +def setup() -> SystemCard: + system_card = SystemCard() + return system_card + + +def test_get_system_card(setup: SystemCard) -> None: + system_card = setup + expected = {"title": None} + + assert system_card.model_dump() == expected + + +def test_system_card_update(setup: SystemCard) -> None: + system_card = setup + expected = {"title": "IAMA 1.1"} + system_card.title = "IAMA 1.1" + + assert system_card.model_dump() == expected