From a10f1e5c0e5aeb52aff4aeb78dcc5ae797aca267 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 14:54:50 -0400 Subject: [PATCH 01/13] work on history tables --- pepdbagent/db_utils.py | 52 ++++++++++++++++++++++++++++++++ pepdbagent/modules/project.py | 56 ++++++++++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 1d0e822..ff9f4c6 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,6 +1,7 @@ import datetime import logging from typing import List, Optional +import enum from sqlalchemy import ( TIMESTAMP, @@ -19,6 +20,7 @@ from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship +from sqlalchemy import Enum from pepdbagent.const import PKG_NAME, POSTGRES_DIALECT from pepdbagent.exceptions import SchemaError @@ -119,6 +121,10 @@ class Projects(Base): namespace_mapping: Mapped["User"] = relationship("User", back_populates="projects_mapping") + history_mapping: Mapped[List["HistoryProjects"]] = relationship( + back_populates="project_mapping", cascade="all, delete-orphan" + ) # TODO: check if cascade is correct + __table_args__ = (UniqueConstraint("namespace", "name", "tag"),) @@ -245,6 +251,52 @@ class ViewSampleAssociation(Base): view: Mapped["Views"] = relationship(back_populates="samples") +class HistoryProjects(Base): + + __tablename__ = "project_history" + + id: Mapped[int] = mapped_column(primary_key=True) + project_id: Mapped[int] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) + user: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="SET NULL")) + update_time: Mapped[datetime.datetime] = mapped_column( + TIMESTAMP(timezone=True), default=deliver_update_date + ) + project_yaml: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) + + project_mapping: Mapped["Projects"] = relationship( + "Projects", back_populates="history_mapping" + ) # TODO: check if cascade is correct + sample_changes_mapping: Mapped[List["HistorySamples"]] = relationship( + back_populates="history_project_mapping", cascade="all, delete-orphan" + ) + + +class UpdateTypes(enum.Enum): + """ + Enum for the type of update + """ + + UPDATE = "update" + INSERT = "insert" + DELETE = "delete" + + +class HistorySamples(Base): + + __tablename__ = "sample_history" + + id: Mapped[int] = mapped_column(primary_key=True) + history_id: Mapped[int] = mapped_column(ForeignKey("project_history.id", ondelete="CASCADE")) + guid: Mapped[str] = mapped_column(nullable=False) + parent_guid: Mapped[Optional[str]] = mapped_column(nullable=True) + sample_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) + change_type: Mapped[UpdateTypes] = mapped_column(Enum(UpdateTypes), nullable=False) + + history_project_mapping: Mapped["HistoryProjects"] = relationship( + "HistoryProjects", back_populates="sample_changes_mapping" + ) + + class BaseEngine: """ A class with base methods, that are used in several classes. e.g. fetch_one or fetch_all diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 1f07a3c..e487379 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -18,7 +18,7 @@ from sqlalchemy.orm.attributes import flag_modified from pepdbagent.const import DEFAULT_TAG, DESCRIPTION_KEY, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, Samples, Subsamples, User +from pepdbagent.db_utils import BaseEngine, Projects, Samples, Subsamples, User, HistoryProjects, HistorySamples, UpdateTypes from pepdbagent.exceptions import ( PEPDatabaseAgentError, ProjectDuplicatedSampleGUIDsError, @@ -481,6 +481,7 @@ def update( namespace: str, name: str, tag: str = DEFAULT_TAG, + user: str = None, ) -> None: """ Update partial parts of the record in db @@ -502,6 +503,7 @@ def update( :param namespace: project namespace :param name: project name :param tag: project tag + :user: user that updates the project if user is not provided, user will be set as Namespace :return: None """ if self.exists(namespace=namespace, name=name, tag=tag): @@ -559,12 +561,21 @@ def update( f"Please provide it to update samples, or use overwrite method." ) + new_history = HistoryProjects( + project_id=found_prj.id, + user=user or namespace, + project_yaml=update_dict["config"], + ) + session.add(new_history) + self._update_samples( project_id=found_prj.id, samples_list=update_dict["samples"], sample_name_key=update_dict["config"].get( SAMPLE_TABLE_INDEX_KEY, "sample_name" ), + history_sa_model=new_history, + ) if "subsamples" in update_dict: @@ -591,6 +602,7 @@ def _update_samples( project_id: int, samples_list: List[Dict[str, str]], sample_name_key: str = "sample_name", + history_sa_model: HistoryProjects = None, ) -> None: """ Update samples in the project @@ -600,6 +612,7 @@ def _update_samples( :param project_id: project id in PEPhub database :param samples_list: list of samples to be updated :param sample_name_key: key of the sample name + :param history_sa_model: HistoryProjects object, to write to the history table :return: None """ @@ -650,18 +663,59 @@ def _update_samples( ) session.add(new_sample) + if history_sa_model: + history_sa_model.sample_changes_mapping.append(HistorySamples( + guid=new_sample.guid, + parent_guid=new_sample.parent_guid, + sample_json=new_sample.sample, + change_type=UpdateTypes.INSERT, + ) + ) + else: + current_history = None if old_samples_mapping[current_id].sample != sample_value: + + if history_sa_model: + current_history = HistorySamples( + guid=old_samples_mapping[current_id].guid, + parent_guid=old_samples_mapping[current_id].parent_guid, + sample_json=old_samples_mapping[current_id].sample, + change_type=UpdateTypes.UPDATE, + ) + old_samples_mapping[current_id].sample = sample_value old_samples_mapping[current_id].sample_name = sample_value[sample_name_key] if old_samples_mapping[current_id].parent_guid != parent_id: + if history_sa_model: + if current_history: + current_history.parent_guid = parent_id + else: + current_history = HistorySamples( + guid=old_samples_mapping[current_id].guid, + parent_guid=old_samples_mapping[current_id].parent_guid, + sample_json=old_samples_mapping[current_id].sample, + change_type=UpdateTypes.UPDATE, + ) old_samples_mapping[current_id].parent_mapping = parent_mapping + if history_sa_model and current_history: + history_sa_model.sample_changes_mapping.append(current_history) + parent_id = current_id parent_mapping = new_sample or old_samples_mapping[current_id] for remove_id in deleted_ids: + + if history_sa_model: + history_sa_model.sample_changes_mapping.append(HistorySamples( + guid=old_samples_mapping[remove_id].guid, + parent_guid=old_samples_mapping[remove_id].parent_guid, + sample_json=old_samples_mapping[remove_id].sample, + change_type=UpdateTypes.DELETE, + ) + ) session.delete(old_samples_mapping[remove_id]) session.commit() From 5054510f4efb6a051a79a0289885fcfede7cfdfc Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 19:06:32 -0400 Subject: [PATCH 02/13] Finished main code for history --- pepdbagent/exceptions.py | 5 + pepdbagent/modules/project.py | 193 ++++++++++++++++++++++++++++++++-- tests/test_project_history.py | 0 3 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 tests/test_project_history.py diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 741de6d..8aa6a89 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -107,3 +107,8 @@ def __init__(self, msg=""): class NamespaceNotFoundError(PEPDatabaseAgentError): def __init__(self, msg=""): super().__init__(f"""Project does not exist. {msg}""") + + +class HistoryNotFoundError(PEPDatabaseAgentError): + def __init__(self, msg=""): + super().__init__(f"""History does not exist. {msg}""") diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index e487379..565e414 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -18,13 +18,23 @@ from sqlalchemy.orm.attributes import flag_modified from pepdbagent.const import DEFAULT_TAG, DESCRIPTION_KEY, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, Samples, Subsamples, User, HistoryProjects, HistorySamples, UpdateTypes +from pepdbagent.db_utils import ( + BaseEngine, + Projects, + Samples, + Subsamples, + User, + HistoryProjects, + HistorySamples, + UpdateTypes, +) from pepdbagent.exceptions import ( PEPDatabaseAgentError, ProjectDuplicatedSampleGUIDsError, ProjectNotFoundError, ProjectUniqueNameError, SampleTableUpdateError, + HistoryNotFoundError, ) from pepdbagent.models import ProjectDict, UpdateItems, UpdateModel from pepdbagent.utils import create_digest, generate_guid, order_samples, registry_path_converter @@ -127,6 +137,31 @@ def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dic :param prj_id: project id :param with_id: retrieve sample with id """ + result_dict = self._get_samples_dict(prj_id, session, with_id) + + result_dict = order_samples(result_dict) + + ordered_samples_list = [sample["sample"] for sample in result_dict] + return ordered_samples_list + + @staticmethod + def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> Dict: + """ + Get not ordered samples from the project. This method is used to retrieve samples from the project + + :param prj_id: project id + :param session: open session object + :param with_id: retrieve sample with id + + :return: dictionary with samples: + {guid: + { + "sample": sample_dict, + "guid": guid, + "parent_guid": parent_guid + } + } + """ samples_results = session.scalars(select(Samples).where(Samples.project_id == prj_id)) result_dict = {} for sample in samples_results: @@ -140,10 +175,7 @@ def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dic "parent_guid": sample.parent_guid, } - result_dict = order_samples(result_dict) - - ordered_samples_list = [sample["sample"] for sample in result_dict] - return ordered_samples_list + return result_dict @staticmethod def _create_select_statement(name: str, namespace: str, tag: str = DEFAULT_TAG) -> Select: @@ -575,7 +607,6 @@ def update( SAMPLE_TABLE_INDEX_KEY, "sample_name" ), history_sa_model=new_history, - ) if "subsamples" in update_dict: @@ -664,7 +695,8 @@ def _update_samples( session.add(new_sample) if history_sa_model: - history_sa_model.sample_changes_mapping.append(HistorySamples( + history_sa_model.sample_changes_mapping.append( + HistorySamples( guid=new_sample.guid, parent_guid=new_sample.parent_guid, sample_json=new_sample.sample, @@ -709,7 +741,8 @@ def _update_samples( for remove_id in deleted_ids: if history_sa_model: - history_sa_model.sample_changes_mapping.append(HistorySamples( + history_sa_model.sample_changes_mapping.append( + HistorySamples( guid=old_samples_mapping[remove_id].guid, parent_guid=old_samples_mapping[remove_id].parent_guid, sample_json=old_samples_mapping[remove_id].sample, @@ -1024,3 +1057,147 @@ def get_samples( .sample_table.replace({np.nan: None}) .to_dict(orient="records") ) + + def get_project_history(self, namespace: str, name: str, tag: str) -> Union[dict, None]: + """ + Get project history annotation by providing namespace, name, and tag + + :param namespace: project namespace + :param name: project name + :param tag: project tag + + :return: project history annotation + """ + + with Session(self._sa_engine) as session: + statement = ( + select(HistoryProjects) + .where( + HistoryProjects.project_id + == select(Projects.id) + .where( + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) + ) + .subquery() + ) + .order_by(HistoryProjects.update_time.desc()) + ) + results = session.scalars(statement) + + # TODO: it's not working, need to be fixed + if results: + return {result.id: result.__dict__ for result in results} + + def get_project_from_history( + self, namespace: str, name: str, tag: str, history_id: int, raw: bool = True + ) -> Union[dict, peppy.Project]: + """ + Get project sample history annotation by providing namespace, name, and tag + + :param namespace: project namespace + :param name: project name + :param tag: project tag + :param history_id: history id + :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] + + :return: project sample history annotation + """ + + with Session(self._sa_engine) as session: + project_mapping = session.scalar( + select(Projects).where( + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) + ) + ) + if not project_mapping: + raise ProjectNotFoundError( + f"No project found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid namespace and project?" + ) + + sample_dict = self._get_samples_dict( + prj_id=project_mapping.id, session=session, with_id=True + ) + + changes_mappings = session.scalars( + select(HistoryProjects) + .where( + and_( + HistoryProjects.project_id == project_mapping.id, + ) + ) + .order_by(HistoryProjects.update_time.desc()) + ) + + changes_ids = [result.id for result in changes_mappings] + + if history_id not in changes_ids: + raise HistoryNotFoundError( + f"No history found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid history id?" + ) + + # Changes mapping is a ordered list from most early to latest changes + # We have to loop through each change and apply it to the sample list + # It should be done before we found the history_id that user is looking for + project_config = None + + for result in changes_mappings: + sample_dict = self._apply_history_changes(sample_dict, result) + + if result.id == history_id: + project_config = result.project_yaml + break + + samples_list = order_samples(sample_dict) + ordered_samples_list = [sample["sample"] for sample in samples_list] + + if raw: + return { + CONFIG_KEY: project_config or project_mapping.config, + SAMPLE_RAW_DICT_KEY: ordered_samples_list, + SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + } + return peppy.Project.from_dict( + pep_dictionary={ + CONFIG_KEY: project_config or project_mapping.config, + SAMPLE_RAW_DICT_KEY: ordered_samples_list, + SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + } + ) + + @staticmethod + def _apply_history_changes(sample_list: dict, change: HistoryProjects) -> dict: + """ + Apply changes from the history to the sample list + + :param sample_list: dictionary with samples + :param change: history change + :return: updated sample list + """ + for sample_change in change.sample_changes_mapping: + sample_id = sample_change.guid + + if sample_change.change_type == UpdateTypes.UPDATE: + sample_list[sample_id]["sample"] = sample_change.sample_json + sample_list[sample_id]["parent_guid"] = sample_change.parent_guid + + elif sample_change.change_type == UpdateTypes.DELETE: + sample_list[sample_id] = { + "sample": sample_change.sample_json, + "guid": sample_id, + "parent_guid": sample_change.parent_guid, + } + + elif sample_change.change_type == UpdateTypes.INSERT: + del sample_list[sample_id] + + return sample_list diff --git a/tests/test_project_history.py b/tests/test_project_history.py new file mode 100644 index 0000000..e69de29 From a6007502c43cc86c790c589d3537fa2203014157 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 19:10:10 -0400 Subject: [PATCH 03/13] updated naming --- pepdbagent/modules/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 565e414..ede3146 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1058,7 +1058,7 @@ def get_samples( .to_dict(orient="records") ) - def get_project_history(self, namespace: str, name: str, tag: str) -> Union[dict, None]: + def get_history(self, namespace: str, name: str, tag: str) -> Union[dict, None]: """ Get project history annotation by providing namespace, name, and tag From 1751d1162fd8fbe86fe89b2a7e3f2ae9e10dbe06 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 19:30:51 -0400 Subject: [PATCH 04/13] small history improvements --- pepdbagent/models.py | 22 ++++++++++++++++++++++ pepdbagent/modules/project.py | 27 +++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 52b8fa6..a6d81cc 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -3,6 +3,7 @@ from peppy.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY from pydantic import BaseModel, ConfigDict, Field, field_validator +import datetime from pepdbagent.const import DEFAULT_TAG @@ -224,3 +225,24 @@ class NamespaceStats(BaseModel): namespace: Union[str, None] = None projects_updated: Dict[str, int] = None projects_created: Dict[str, int] = None + + +class HistoryChangeModel(BaseModel): + """ + Model for history change + """ + + change_id: int + change_date: datetime.datetime + user: str + + +class HistoryAnnotationModel(BaseModel): + """ + History annotation model + """ + + namespace: str + name: str + tag: str = DEFAULT_TAG + history: List[HistoryChangeModel] diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index ede3146..82f758b 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -36,7 +36,13 @@ SampleTableUpdateError, HistoryNotFoundError, ) -from pepdbagent.models import ProjectDict, UpdateItems, UpdateModel +from pepdbagent.models import ( + ProjectDict, + UpdateItems, + UpdateModel, + HistoryChangeModel, + HistoryAnnotationModel, +) from pepdbagent.utils import create_digest, generate_guid, order_samples, registry_path_converter _LOGGER = logging.getLogger(PKG_NAME) @@ -1058,7 +1064,7 @@ def get_samples( .to_dict(orient="records") ) - def get_history(self, namespace: str, name: str, tag: str) -> Union[dict, None]: + def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationModel: """ Get project history annotation by providing namespace, name, and tag @@ -1087,10 +1093,23 @@ def get_history(self, namespace: str, name: str, tag: str) -> Union[dict, None]: .order_by(HistoryProjects.update_time.desc()) ) results = session.scalars(statement) + return_results: List = [] - # TODO: it's not working, need to be fixed if results: - return {result.id: result.__dict__ for result in results} + for result in results: + return_results.append( + HistoryChangeModel( + change_id=result.id, + change_date=result.update_time, + user=result.user, + ) + ) + return HistoryAnnotationModel( + project_name=name, + project_namespace=namespace, + project_tag=tag, + history=return_results, + ) def get_project_from_history( self, namespace: str, name: str, tag: str, history_id: int, raw: bool = True From 0e4c1e5d9a04a294d717a18daf4926154651ba09 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 19:44:05 -0400 Subject: [PATCH 05/13] bug fixes --- pepdbagent/modules/project.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 82f758b..95fded7 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1105,9 +1105,9 @@ def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationM ) ) return HistoryAnnotationModel( - project_name=name, - project_namespace=namespace, - project_tag=tag, + namespace=namespace, + name=name, + tag=tag, history=return_results, ) @@ -1146,23 +1146,31 @@ def get_project_from_history( prj_id=project_mapping.id, session=session, with_id=True ) - changes_mappings = session.scalars( + main_history = session.scalar( select(HistoryProjects) .where( and_( HistoryProjects.project_id == project_mapping.id, + HistoryProjects.id == history_id, ) ) .order_by(HistoryProjects.update_time.desc()) ) - - changes_ids = [result.id for result in changes_mappings] - - if history_id not in changes_ids: + if not main_history: raise HistoryNotFoundError( - f"No history found for supplied input: '{namespace}/{name}:{tag}'. " - f"Did you supply a valid history id?" + f"No history found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid history id?" + ) + + changes_mappings = session.scalars( + select(HistoryProjects) + .where( + and_( + HistoryProjects.project_id == project_mapping.id, + ) ) + .order_by(HistoryProjects.update_time.desc()) + ) # Changes mapping is a ordered list from most early to latest changes # We have to loop through each change and apply it to the sample list From 73030b4ef28704b22f4fe12c56515f97b3629de8 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 3 Jul 2024 19:45:38 -0400 Subject: [PATCH 06/13] lint --- pepdbagent/modules/project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 95fded7..76cd455 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1158,9 +1158,9 @@ def get_project_from_history( ) if not main_history: raise HistoryNotFoundError( - f"No history found for supplied input: '{namespace}/{name}:{tag}'. " - f"Did you supply a valid history id?" - ) + f"No history found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid history id?" + ) changes_mappings = session.scalars( select(HistoryProjects) From cbcfc4c63b10a0d06b9367c5bbb51cc29a438bd9 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 5 Jul 2024 10:50:58 -0400 Subject: [PATCH 07/13] added with_id argument --- pepdbagent/modules/project.py | 44 ++++++++++++++++-------- tests/test_project_history.py | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 76cd455..1480c99 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -681,6 +681,19 @@ def _update_samples( del new_samples_ids_list, new_samples_ids_set + for remove_id in deleted_ids: + + if history_sa_model: + history_sa_model.sample_changes_mapping.append( + HistorySamples( + guid=old_samples_mapping[remove_id].guid, + parent_guid=old_samples_mapping[remove_id].parent_guid, + sample_json=old_samples_mapping[remove_id].sample, + change_type=UpdateTypes.DELETE, + ) + ) + session.delete(old_samples_mapping[remove_id]) + parent_id = None parent_mapping = None @@ -744,19 +757,6 @@ def _update_samples( parent_id = current_id parent_mapping = new_sample or old_samples_mapping[current_id] - for remove_id in deleted_ids: - - if history_sa_model: - history_sa_model.sample_changes_mapping.append( - HistorySamples( - guid=old_samples_mapping[remove_id].guid, - parent_guid=old_samples_mapping[remove_id].parent_guid, - sample_json=old_samples_mapping[remove_id].sample, - change_type=UpdateTypes.DELETE, - ) - ) - session.delete(old_samples_mapping[remove_id]) - session.commit() @staticmethod @@ -1088,7 +1088,7 @@ def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationM Projects.tag == tag, ) ) - .subquery() + .scalar_subquery() ) .order_by(HistoryProjects.update_time.desc()) ) @@ -1112,7 +1112,13 @@ def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationM ) def get_project_from_history( - self, namespace: str, name: str, tag: str, history_id: int, raw: bool = True + self, + namespace: str, + name: str, + tag: str, + history_id: int, + raw: bool = True, + with_id: bool = False, ) -> Union[dict, peppy.Project]: """ Get project sample history annotation by providing namespace, name, and tag @@ -1122,6 +1128,7 @@ def get_project_from_history( :param tag: project tag :param history_id: history id :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] + :param with_id: if True, retrieve samples with ids. [Default: False] :return: project sample history annotation """ @@ -1187,6 +1194,13 @@ def get_project_from_history( samples_list = order_samples(sample_dict) ordered_samples_list = [sample["sample"] for sample in samples_list] + if not with_id: + for sample in ordered_samples_list: + try: + del sample[PEPHUB_SAMPLE_ID_KEY] + except KeyError: + pass + if raw: return { CONFIG_KEY: project_config or project_mapping.config, diff --git a/tests/test_project_history.py b/tests/test_project_history.py index e69de29..3b05ea0 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -0,0 +1,65 @@ +import numpy as np +import peppy +import pytest + +from pepdbagent.exceptions import ProjectNotFoundError + +from .utils import PEPDBAgentContextManager, get_path_to_example_file, list_of_available_peps + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestProjectHistory: + """ + Test project methods + """ + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_update_history(self, namespace, name, sample_name): ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_insert_history(self, namespace, name, sample_name): ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_delete_history(self, namespace, name, sample_name): ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_correct_order(self, namespace, name, sample_name): ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_history(self, namespace, name, sample_name): ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_correct_order(self, namespace, name, sample_name): ... From a63e982f4cc33c2a8a8ca538996792b84cbf5de9 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 5 Jul 2024 16:28:29 -0400 Subject: [PATCH 08/13] added delete function to history added tests --- pepdbagent/const.py | 2 + pepdbagent/db_utils.py | 4 +- pepdbagent/models.py | 2 +- pepdbagent/modules/project.py | 98 ++++++++++++--- tests/test_annotation.py | 2 + tests/test_namespace.py | 1 + tests/test_project_history.py | 216 ++++++++++++++++++++++++++++++++-- tests/test_samples.py | 1 + tests/test_updates.py | 3 +- tests/utils.py | 3 +- 10 files changed, 302 insertions(+), 30 deletions(-) diff --git a/pepdbagent/const.py b/pepdbagent/const.py index 055397a..aae67f6 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -20,3 +20,5 @@ LAST_UPDATE_DATE_KEY = "last_update_date" PEPHUB_SAMPLE_ID_KEY = "ph_id" + +MAX_HISTORY_SAMPLES_NUMBER = 2000 diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index ff9f4c6..5b50149 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,11 +1,12 @@ import datetime +import enum import logging from typing import List, Optional -import enum from sqlalchemy import ( TIMESTAMP, BigInteger, + Enum, FetchedValue, ForeignKey, Result, @@ -20,7 +21,6 @@ from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship -from sqlalchemy import Enum from pepdbagent.const import PKG_NAME, POSTGRES_DIALECT from pepdbagent.exceptions import SchemaError diff --git a/pepdbagent/models.py b/pepdbagent/models.py index a6d81cc..c138d18 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -1,9 +1,9 @@ # file with pydantic models +import datetime from typing import Dict, List, Optional, Union from peppy.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY from pydantic import BaseModel, ConfigDict, Field, field_validator -import datetime from pepdbagent.const import DEFAULT_TAG diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 1480c99..fd50ec9 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -17,31 +17,38 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import DEFAULT_TAG, DESCRIPTION_KEY, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME +from pepdbagent.const import ( + DEFAULT_TAG, + DESCRIPTION_KEY, + MAX_HISTORY_SAMPLES_NUMBER, + NAME_KEY, + PEPHUB_SAMPLE_ID_KEY, + PKG_NAME, +) from pepdbagent.db_utils import ( BaseEngine, + HistoryProjects, + HistorySamples, Projects, Samples, Subsamples, - User, - HistoryProjects, - HistorySamples, UpdateTypes, + User, ) from pepdbagent.exceptions import ( + HistoryNotFoundError, PEPDatabaseAgentError, ProjectDuplicatedSampleGUIDsError, ProjectNotFoundError, ProjectUniqueNameError, SampleTableUpdateError, - HistoryNotFoundError, ) from pepdbagent.models import ( + HistoryAnnotationModel, + HistoryChangeModel, ProjectDict, UpdateItems, UpdateModel, - HistoryChangeModel, - HistoryAnnotationModel, ) from pepdbagent.utils import create_digest, generate_guid, order_samples, registry_path_converter @@ -541,7 +548,7 @@ def update( :param namespace: project namespace :param name: project name :param tag: project tag - :user: user that updates the project if user is not provided, user will be set as Namespace + :param user: user that updates the project if user is not provided, user will be set as Namespace :return: None """ if self.exists(namespace=namespace, name=name, tag=tag): @@ -598,13 +605,19 @@ def update( f"pephub_sample_id '{PEPHUB_SAMPLE_ID_KEY}' is missing in samples." f"Please provide it to update samples, or use overwrite method." ) - - new_history = HistoryProjects( - project_id=found_prj.id, - user=user or namespace, - project_yaml=update_dict["config"], - ) - session.add(new_history) + if len(update_dict["samples"]) > MAX_HISTORY_SAMPLES_NUMBER: + _LOGGER.warning( + f"Number of samples in the project exceeds the limit of {MAX_HISTORY_SAMPLES_NUMBER}." + f"Samples won't be updated." + ) + new_history = None + else: + new_history = HistoryProjects( + project_id=found_prj.id, + user=user or namespace, + project_yaml=update_dict["config"], + ) + session.add(new_history) self._update_samples( project_id=found_prj.id, @@ -639,7 +652,7 @@ def _update_samples( project_id: int, samples_list: List[Dict[str, str]], sample_name_key: str = "sample_name", - history_sa_model: HistoryProjects = None, + history_sa_model: Union[HistoryProjects, None] = None, ) -> None: """ Update samples in the project @@ -1242,3 +1255,56 @@ def _apply_history_changes(sample_list: dict, change: HistoryProjects) -> dict: del sample_list[sample_id] return sample_list + + def delete_history( + self, namespace: str, name: str, tag: str, history_id: Union[int, None] = None + ) -> None: + """ + Delete history from the project + + :param namespace: project namespace + :param name: project name + :param tag: project tag + :param history_id: history id. If none is provided, all history will be deleted + + :return: None + """ + with Session(self._sa_engine) as session: + project_mapping = session.scalar( + select(Projects).where( + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) + ) + ) + if not project_mapping: + raise ProjectNotFoundError( + f"No project found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid namespace and project?" + ) + + if history_id is None: + session.execute( + delete(HistoryProjects).where(HistoryProjects.project_id == project_mapping.id) + ) + session.commit() + return None + + history_mapping = session.scalar( + select(HistoryProjects).where( + and_( + HistoryProjects.project_id == project_mapping.id, + HistoryProjects.id == history_id, + ) + ) + ) + if not history_mapping: + raise HistoryNotFoundError( + f"No history found for supplied input: '{namespace}/{name}:{tag}'. " + f"Did you supply a valid history id?" + ) + + session.delete(history_mapping) + session.commit() diff --git a/tests/test_annotation.py b/tests/test_annotation.py index e6778e3..9e5603e 100644 --- a/tests/test_annotation.py +++ b/tests/test_annotation.py @@ -1,5 +1,7 @@ import datetime + import pytest + from pepdbagent.exceptions import FilterError, ProjectNotFoundError from .utils import PEPDBAgentContextManager diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 432a9c2..83fa4d1 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -1,4 +1,5 @@ import pytest + from pepdbagent.exceptions import ProjectAlreadyInFavorites, ProjectNotInFavorites from .utils import PEPDBAgentContextManager diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 3b05ea0..9399c39 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -1,10 +1,10 @@ -import numpy as np import peppy import pytest -from pepdbagent.exceptions import ProjectNotFoundError +from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY +from pepdbagent.exceptions import HistoryNotFoundError -from .utils import PEPDBAgentContextManager, get_path_to_example_file, list_of_available_peps +from .utils import PEPDBAgentContextManager @pytest.mark.skipif( @@ -22,7 +22,38 @@ class TestProjectHistory: ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_update_history(self, namespace, name, sample_name): ... + def test_get_add_history_all_annotation(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + + del prj["_sample_dict"][1] + del prj["_sample_dict"][2] + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + new_sample2 = { + "sample_name": "new_sample2", + "protocol": "new_protocol2", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample1.copy()) + prj["_sample_dict"].append(new_sample2.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + project_history = agent.project.get_history(namespace, name, tag="default") + + assert len(project_history.history) == 1 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -30,7 +61,40 @@ def test_get_update_history(self, namespace, name, sample_name): ... ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_insert_history(self, namespace, name, sample_name): ... + def test_get_add_history_all_project(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj_init = agent.project.get(namespace, name, tag="default", raw=False) + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + + del prj["_sample_dict"][1] + del prj["_sample_dict"][2] + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + new_sample2 = { + "sample_name": "new_sample2", + "protocol": "new_protocol2", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample1.copy()) + prj["_sample_dict"].append(new_sample2.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + history_prj = agent.project.get_project_from_history( + namespace, name, tag="default", history_id=1, raw=False + ) + assert prj_init == history_prj @pytest.mark.parametrize( "namespace, name, sample_name", @@ -38,7 +102,38 @@ def test_get_insert_history(self, namespace, name, sample_name): ... ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_delete_history(self, namespace, name, sample_name): ... + def test_get_history_multiple_changes(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + prj["_sample_dict"].append(new_sample1.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + history = agent.project.get_history(namespace, name, tag="default") + + assert len(history.history) == 2 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -46,7 +141,22 @@ def test_get_delete_history(self, namespace, name, sample_name): ... ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_correct_order(self, namespace, name, sample_name): ... + def test_get_project_incorrect_history_id(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, tag="default", with_id=True) + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + with pytest.raises(HistoryNotFoundError): + agent.project.get_project_from_history( + namespace, "amendments2", tag="default", history_id=1, raw=False + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -54,7 +164,10 @@ def test_get_correct_order(self, namespace, name, sample_name): ... ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_history(self, namespace, name, sample_name): ... + def test_get_history_none(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + history_annot = agent.project.get_history(namespace, name, tag="default") + assert len(history_annot.history) == 0 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -62,4 +175,89 @@ def test_get_history(self, namespace, name, sample_name): ... ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_correct_order(self, namespace, name, sample_name): ... + def test_delete_all_history(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + prj["_sample_dict"].append(new_sample1.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + history = agent.project.get_history(namespace, name, tag="default") + + assert len(history.history) == 2 + + agent.project.delete_history(namespace, name, tag="default", history_id=None) + + history = agent.project.get_history(namespace, name, tag="default") + assert len(history.history) == 0 + + project_exists = agent.project.exists(namespace, name, tag="default") + assert project_exists + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_delete_one_history(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + prj["_sample_dict"].append(new_sample1.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + history = agent.project.get_history(namespace, name, tag="default") + + assert len(history.history) == 2 + + agent.project.delete_history(namespace, name, tag="default", history_id=1) + + history = agent.project.get_history(namespace, name, tag="default") + + assert len(history.history) == 1 + assert history.history[0].change_id == 2 diff --git a/tests/test_samples.py b/tests/test_samples.py index 0e4de49..e8a6862 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -1,5 +1,6 @@ import peppy import pytest + from pepdbagent.exceptions import SampleNotFoundError from .utils import PEPDBAgentContextManager diff --git a/tests/test_updates.py b/tests/test_updates.py index 73854c9..790b313 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -1,8 +1,9 @@ import peppy import pytest from peppy.exceptions import IllegalStateException -from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError + from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY +from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError from .utils import PEPDBAgentContextManager diff --git a/tests/utils.py b/tests/utils.py index 6e3e7f4..8ddc820 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,7 @@ import os -import peppy import warnings + +import peppy from sqlalchemy.exc import OperationalError from pepdbagent import PEPDatabaseAgent From 7b469fea48d0348dba780f7607b9cdd914b0ac41 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 8 Jul 2024 12:17:08 -0400 Subject: [PATCH 09/13] bug workaround in history table --- pepdbagent/modules/project.py | 10 +++++++++- tests/test_project_history.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index fd50ec9..2b43f5f 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -670,6 +670,12 @@ def _update_samples( old_samples = session.scalars(select(Samples).where(Samples.project_id == project_id)) old_samples_mapping: dict = {sample.guid: sample for sample in old_samples} + + # old_child_parent_id needed because of the parent_guid is sometimes set to none in sqlalchemy mapping :( bug + old_child_parent_id: Dict[str, str] = { + child: mapping.parent_guid for child, mapping in old_samples_mapping.items() + } + old_samples_ids_set: set = set(old_samples_mapping.keys()) new_samples_ids_list: list = [ new_sample[PEPHUB_SAMPLE_ID_KEY] @@ -751,6 +757,8 @@ def _update_samples( old_samples_mapping[current_id].sample = sample_value old_samples_mapping[current_id].sample_name = sample_value[sample_name_key] + # !bug workaround: if project was deleted and sometimes old_samples_mapping[current_id].parent_guid + # and it can cause an error in history. For this we have `old_child_parent_id` dict if old_samples_mapping[current_id].parent_guid != parent_id: if history_sa_model: if current_history: @@ -758,7 +766,7 @@ def _update_samples( else: current_history = HistorySamples( guid=old_samples_mapping[current_id].guid, - parent_guid=old_samples_mapping[current_id].parent_guid, + parent_guid=old_child_parent_id[current_id], sample_json=old_samples_mapping[current_id].sample, change_type=UpdateTypes.UPDATE, ) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 9399c39..d5e09fe 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -66,7 +66,7 @@ def test_get_add_history_all_project(self, namespace, name, sample_name): prj_init = agent.project.get(namespace, name, tag="default", raw=False) prj = agent.project.get(namespace, name, tag="default", with_id=True) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + # prj["_sample_dict"][0]["sample_name"] = "new_sample_name" del prj["_sample_dict"][1] del prj["_sample_dict"][2] From d5416233a6982d0d7d1bbbdfd7596b8b2ceb8753 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 8 Jul 2024 12:51:43 -0400 Subject: [PATCH 10/13] removed row_number from database --- pepdbagent/db_utils.py | 1 - pepdbagent/modules/project.py | 4 +--- pepdbagent/modules/sample.py | 1 - tests/test_project.py | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 5b50149..378c303 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -137,7 +137,6 @@ class Samples(Base): id: Mapped[int] = mapped_column(primary_key=True) sample: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) - row_number: Mapped[int] # TODO: should be removed project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) project_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") sample_name: Mapped[Optional[str]] = mapped_column() diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 2b43f5f..c99e1f5 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -726,7 +726,6 @@ def _update_samples( sample=sample_value, guid=current_id, sample_name=sample_value[sample_name_key], - row_number=0, project_id=project_id, parent_mapping=parent_mapping, ) @@ -895,11 +894,10 @@ def _add_samples_to_project( :return: NoReturn """ previous_sample_guid = None - for row_number, sample in enumerate(samples): + for sample in samples: sample = Samples( sample=sample, - row_number=row_number, sample_name=sample.get(sample_table_index), parent_guid=previous_sample_guid, guid=generate_guid(), diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index 7e8d89a..9f39760 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -234,7 +234,6 @@ def add( else: sample_mapping = Samples( sample=sample_dict, - row_number=0, project_id=project_mapping.id, sample_name=sample_name, guid=generate_guid(), diff --git a/tests/test_project.py b/tests/test_project.py index 26cf017..6524bcc 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -19,7 +19,7 @@ class TestProject: def test_create_project(self): with PEPDBAgentContextManager(add_data=False) as agent: prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) - agent.project.create(prj, namespace="test", name="imply", overwrite=True) + agent.project.create(prj, namespace="test", name="imply", overwrite=False) assert True def test_create_project_from_dict(self): From 6490e23815544b771c29e5ad2b7540f264fd4d11 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 8 Jul 2024 13:03:33 -0400 Subject: [PATCH 11/13] fixed PR comments --- pepdbagent/db_utils.py | 2 +- pepdbagent/modules/project.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 378c303..ea36bd8 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -264,7 +264,7 @@ class HistoryProjects(Base): project_mapping: Mapped["Projects"] = relationship( "Projects", back_populates="history_mapping" - ) # TODO: check if cascade is correct + ) sample_changes_mapping: Mapped[List["HistorySamples"]] = relationship( back_populates="history_project_mapping", cascade="all, delete-orphan" ) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index c99e1f5..1e63992 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1235,11 +1235,11 @@ def get_project_from_history( ) @staticmethod - def _apply_history_changes(sample_list: dict, change: HistoryProjects) -> dict: + def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: """ Apply changes from the history to the sample list - :param sample_list: dictionary with samples + :param sample_dict: dictionary with samples :param change: history change :return: updated sample list """ @@ -1247,20 +1247,20 @@ def _apply_history_changes(sample_list: dict, change: HistoryProjects) -> dict: sample_id = sample_change.guid if sample_change.change_type == UpdateTypes.UPDATE: - sample_list[sample_id]["sample"] = sample_change.sample_json - sample_list[sample_id]["parent_guid"] = sample_change.parent_guid + sample_dict[sample_id]["sample"] = sample_change.sample_json + sample_dict[sample_id]["parent_guid"] = sample_change.parent_guid elif sample_change.change_type == UpdateTypes.DELETE: - sample_list[sample_id] = { + sample_dict[sample_id] = { "sample": sample_change.sample_json, "guid": sample_id, "parent_guid": sample_change.parent_guid, } elif sample_change.change_type == UpdateTypes.INSERT: - del sample_list[sample_id] + del sample_dict[sample_id] - return sample_list + return sample_dict def delete_history( self, namespace: str, name: str, tag: str, history_id: Union[int, None] = None From ef6fda4ffc4c3837b959cc17e87e00ee18f5e423 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 8 Jul 2024 14:55:07 -0400 Subject: [PATCH 12/13] Added delete user method --- pepdbagent/exceptions.py | 5 +++++ pepdbagent/modules/user.py | 20 ++++++++++++++++++- tests/test_namespace.py | 39 +++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 8aa6a89..64e5278 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -112,3 +112,8 @@ def __init__(self, msg=""): class HistoryNotFoundError(PEPDatabaseAgentError): def __init__(self, msg=""): super().__init__(f"""History does not exist. {msg}""") + + +class UserNotFoundError(PEPDatabaseAgentError): + def __init__(self, msg=""): + super().__init__(f"""User does not exist. {msg}""") diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index eea8f84..125c7c1 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -7,7 +7,11 @@ from pepdbagent.const import PKG_NAME from pepdbagent.db_utils import BaseEngine, Projects, Stars, User -from pepdbagent.exceptions import ProjectAlreadyInFavorites, ProjectNotInFavorites +from pepdbagent.exceptions import ( + ProjectAlreadyInFavorites, + ProjectNotInFavorites, + UserNotFoundError, +) from pepdbagent.models import AnnotationList, AnnotationModel _LOGGER = logging.getLogger(PKG_NAME) @@ -214,3 +218,17 @@ def exists( return True else: return False + + def delete(self, namespace: str) -> None: + """ + Delete user from the database with all related data + + :param namespace: user namespace + :return: None + """ + if not self.exists(namespace): + raise UserNotFoundError + + with Session(self._sa_engine) as session: + session.execute(delete(User).where(User.namespace == namespace)) + session.commit() diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 83fa4d1..196f6e8 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -63,7 +63,7 @@ def test_namespace_stats(self): ) class TestFavorites: """ - Test function within user class + Test method related to favorites """ def test_add_projects_to_favorites(self): @@ -156,3 +156,40 @@ def test_annotation_favorite_number(self, namespace, name): assert prj_annot.stars_number == 1 else: assert prj_annot.stars_number == 0 + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestUser: + """ + Test methods within user class + """ + + def test_create_user(self): + with PEPDBAgentContextManager(add_data=True) as agent: + + user = agent.user.create_user("test_user") + + assert agent.user.exists("test_user") + + def test_delete_user(self): + with PEPDBAgentContextManager(add_data=True) as agent: + + test_user = "test_user" + agent.user.create_user(test_user) + assert agent.user.exists(test_user) + agent.user.delete(test_user) + assert not agent.user.exists(test_user) + + def test_delete_user_deletes_projects(self): + with PEPDBAgentContextManager(add_data=True) as agent: + + test_user = "namespace1" + + assert agent.user.exists(test_user) + agent.user.delete(test_user) + assert not agent.user.exists(test_user) + results = agent.namespace.get(query=test_user) + assert len(results.results) == 0 From 92a9ac67e19dec693486bb2fbd325dfd1e998aab Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 8 Jul 2024 15:34:12 -0400 Subject: [PATCH 13/13] added restore from history method --- pepdbagent/modules/project.py | 36 ++++++++++++++++++++++++++++++ tests/test_project_history.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 1e63992..97384a5 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1314,3 +1314,39 @@ def delete_history( session.delete(history_mapping) session.commit() + + def restore( + self, + namespace: str, + name: str, + tag: str, + history_id: int, + user: str = None, + ) -> None: + """ + Restore project to the specific history state + + :param namespace: project namespace + :param name: project name + :param tag: project tag + :param history_id: history id + :param user: user that restores the project if user is not provided, user will be set as Namespace + + :return: None + """ + + restore_project = self.get_project_from_history( + namespace=namespace, + name=name, + tag=tag, + history_id=history_id, + raw=True, + with_id=True, + ) + self.update( + update_dict={"project": peppy.Project.from_dict(restore_project)}, + namespace=namespace, + name=name, + tag=tag, + user=user or namespace, + ) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index d5e09fe..332e1ca 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -261,3 +261,45 @@ def test_delete_one_history(self, namespace, name, sample_name): assert len(history.history) == 1 assert history.history[0].change_id == 2 + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_restore_project(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj_org = agent.project.get(namespace, name, tag="default", with_id=False) + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + prj = agent.project.get(namespace, name, tag="default", with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + prj["_sample_dict"].append(new_sample1.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + agent.project.restore(namespace, name, tag="default", history_id=1) + + restored_project = agent.project.get(namespace, name, tag="default", with_id=False) + + assert prj_org == restored_project