diff --git a/.github/workflows/cli-coverage.yml b/.github/workflows/cli-coverage.yml index de3a3c0..e33fcda 100644 --- a/.github/workflows/cli-coverage.yml +++ b/.github/workflows/cli-coverage.yml @@ -17,7 +17,7 @@ jobs: image: postgres env: POSTGRES_USER: postgres - POSTGRES_PASSWORD: docker + POSTGRES_PASSWORD: pass8743hf9h23f87h437 POSTGRES_DB: pep-db POSTGRES_HOST: localhost ports: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 16f6d0b..e4402e4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,7 +19,7 @@ jobs: image: postgres env: POSTGRES_USER: postgres - POSTGRES_PASSWORD: docker + POSTGRES_PASSWORD: pass8743hf9h23f87h437 POSTGRES_DB: pep-db POSTGRES_HOST: localhost ports: diff --git a/docs/changelog.md b/docs/changelog.md index d1f7d24..ded803c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.9.0] -- 2024-06-25 +- Introduced new sample ordering with linked list [#133](https://github.com/pepkit/pepdbagent/issues/133) +- Efficiency improvements of project update function +- Test restructuring + + ## [0.8.0] -- 2024-02-26 - Fixed forking schema - Improved forking efficiency [#129](https://github.com/pepkit/pepdbagent/issues/129) diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index 777f190..3e2f46a 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.9.0" diff --git a/pepdbagent/const.py b/pepdbagent/const.py index 31819c8..055397a 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -18,3 +18,5 @@ SUBMISSION_DATE_KEY = "submission_date" LAST_UPDATE_DATE_KEY = "last_update_date" + +PEPHUB_SAMPLE_ID_KEY = "ph_id" diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 50a315e..1d0e822 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,32 +1,26 @@ import datetime import logging -from typing import Optional, List +from typing import List, Optional from sqlalchemy import ( + TIMESTAMP, BigInteger, FetchedValue, + ForeignKey, Result, Select, String, + UniqueConstraint, event, select, - TIMESTAMP, - ForeignKey, - UniqueConstraint, ) from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.engine import URL, create_engine from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles -from sqlalchemy.orm import ( - DeclarativeBase, - Mapped, - Session, - mapped_column, - relationship, -) +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship -from pepdbagent.const import POSTGRES_DIALECT, PKG_NAME +from pepdbagent.const import PKG_NAME, POSTGRES_DIALECT from pepdbagent.exceptions import SchemaError _LOGGER = logging.getLogger(PKG_NAME) @@ -77,7 +71,7 @@ class Projects(Base): __tablename__ = "projects" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column() + namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) name: Mapped[str] = mapped_column() tag: Mapped[str] = mapped_column() digest: Mapped[str] = mapped_column(String(32)) @@ -93,7 +87,7 @@ class Projects(Base): pep_schema: Mapped[Optional[str]] pop: Mapped[Optional[bool]] = mapped_column(default=False) samples_mapping: Mapped[List["Samples"]] = relationship( - back_populates="sample_mapping", cascade="all, delete-orphan" + back_populates="project_mapping", cascade="all, delete-orphan" ) subsamples_mapping: Mapped[List["Subsamples"]] = relationship( back_populates="subsample_mapping", cascade="all, delete-orphan" @@ -114,13 +108,17 @@ class Projects(Base): back_populates="forked_to_mapping", remote_side=[id], single_parent=True, - cascade="all", + cascade="save-update, merge, refresh-expire", ) forked_to_mapping = relationship( - "Projects", back_populates="forked_from_mapping", cascade="all" + "Projects", + back_populates="forked_from_mapping", + cascade="save-update, merge, refresh-expire", ) + namespace_mapping: Mapped["User"] = relationship("User", back_populates="projects_mapping") + __table_args__ = (UniqueConstraint("namespace", "name", "tag"),) @@ -133,10 +131,28 @@ class Samples(Base): id: Mapped[int] = mapped_column(primary_key=True) sample: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) - row_number: Mapped[int] + row_number: Mapped[int] # TODO: should be removed project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) + project_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") sample_name: Mapped[Optional[str]] = mapped_column() - sample_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") + guid: Mapped[Optional[str]] = mapped_column(nullable=False, unique=True) + + submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) + last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + default=deliver_update_date, + onupdate=deliver_update_date, + ) + + parent_guid: Mapped[Optional[str]] = mapped_column( + ForeignKey("samples.guid", ondelete="CASCADE"), + nullable=True, + doc="Parent sample id. Used to create a hierarchy of samples.", + ) + + parent_mapping: Mapped["Samples"] = relationship( + "Samples", remote_side=guid, back_populates="child_mapping" + ) + child_mapping: Mapped["Samples"] = relationship("Samples", back_populates="parent_mapping") views: Mapped[Optional[List["ViewSampleAssociation"]]] = relationship( back_populates="sample", cascade="all, delete-orphan" @@ -166,12 +182,17 @@ class User(Base): __tablename__ = "users" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] + namespace: Mapped[str] = mapped_column(nullable=False, unique=True) stars_mapping: Mapped[List["Stars"]] = relationship( back_populates="user_mapping", cascade="all, delete-orphan", order_by="Stars.star_date.desc()", ) + number_of_projects: Mapped[int] = mapped_column(default=0) + + projects_mapping: Mapped[List["Projects"]] = relationship( + "Projects", back_populates="namespace_mapping" + ) class Stars(Base): @@ -318,3 +339,15 @@ def check_db_connection(self): self.session_execute(select(Projects).limit(1)) except ProgrammingError: raise SchemaError() + + def delete_schema(self, engine=None) -> None: + """ + Delete sql schema in the database. + + :param engine: sqlalchemy engine [Default: None] + :return: None + """ + if not engine: + engine = self._engine + Base.metadata.drop_all(engine) + return None diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index a9b0bda..741de6d 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -61,6 +61,16 @@ def __init__(self, msg=""): super().__init__(f"""Sample does not exist. {msg}""") +class SampleTableUpdateError(PEPDatabaseAgentError): + def __init__(self, msg=""): + super().__init__(f"""Sample table update error. {msg}""") + + +class ProjectDuplicatedSampleGUIDsError(SampleTableUpdateError): + def __init__(self, msg=""): + super().__init__(f"""Project has duplicated sample GUIDs. {msg}""") + + class SampleAlreadyExistsError(PEPDatabaseAgentError): def __init__(self, msg=""): super().__init__(f"""Sample already exists. {msg}""") diff --git a/pepdbagent/models.py b/pepdbagent/models.py index a0b2c45..52b8fa6 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -1,7 +1,8 @@ # file with pydantic models -from typing import List, Optional, Union, Dict -from pydantic import BaseModel, Field, ConfigDict, field_validator -from peppy.const import CONFIG_KEY, SUBSAMPLE_RAW_LIST_KEY, SAMPLE_RAW_DICT_KEY +from typing import Dict, List, Optional, Union + +from peppy.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY +from pydantic import BaseModel, ConfigDict, Field, field_validator from pepdbagent.const import DEFAULT_TAG diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index bd9f511..2b30ccb 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -3,16 +3,16 @@ from typing import List, Literal, Optional, Union from sqlalchemy import and_, func, or_, select -from sqlalchemy.sql.selectable import Select from sqlalchemy.orm import Session +from sqlalchemy.sql.selectable import Select from pepdbagent.const import ( DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_TAG, + LAST_UPDATE_DATE_KEY, PKG_NAME, SUBMISSION_DATE_KEY, - LAST_UPDATE_DATE_KEY, ) from pepdbagent.db_utils import BaseEngine, Projects from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 5af92db..5af4c90 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -1,20 +1,20 @@ import logging -from typing import List, Union, Tuple from collections import Counter from datetime import datetime, timedelta +from typing import List, Tuple, Union -from sqlalchemy import distinct, func, or_, select, text -from sqlalchemy.sql.selectable import Select +from sqlalchemy import distinct, func, or_, select from sqlalchemy.orm import Session +from sqlalchemy.sql.selectable import Select -from pepdbagent.const import DEFAULT_LIMIT, DEFAULT_OFFSET, PKG_NAME, DEFAULT_LIMIT_INFO +from pepdbagent.const import DEFAULT_LIMIT, DEFAULT_LIMIT_INFO, DEFAULT_OFFSET, PKG_NAME +from pepdbagent.db_utils import BaseEngine, Projects, User from pepdbagent.exceptions import NamespaceNotFoundError -from pepdbagent.db_utils import Projects, BaseEngine from pepdbagent.models import ( + ListOfNamespaceInfo, Namespace, - NamespaceList, NamespaceInfo, - ListOfNamespaceInfo, + NamespaceList, NamespaceStats, ) from pepdbagent.utils import tuple_converter @@ -172,9 +172,54 @@ def _add_condition( ) return statement + # old function, that counts namespace info based on Projects table + # def info(self, limit: int = DEFAULT_LIMIT_INFO) -> ListOfNamespaceInfo: + # """ + # Get list of top n namespaces in the database + # + # :param limit: limit of results (top namespace ) + # :return: number_of_namespaces: int + # limit: int + # results: { namespace: str + # number_of_projects: int + # } + # """ + # total_number_of_namespaces = self._count_namespace() + # + # statement = ( + # select( + # func.count(Projects.namespace).label("number_of_projects"), + # Projects.namespace, + # ) + # .select_from(Projects) + # .where(Projects.private.is_(False)) + # .limit(limit) + # .order_by(text("number_of_projects desc")) + # .group_by(Projects.namespace) + # ) + # + # with Session(self._sa_engine) as session: + # query_results = session.execute(statement).all() + # + # list_of_results = [] + # for result in query_results: + # list_of_results.append( + # NamespaceInfo( + # namespace=result.namespace, + # number_of_projects=result.number_of_projects, + # ) + # ) + # return ListOfNamespaceInfo( + # number_of_namespaces=total_number_of_namespaces, + # limit=limit, + # results=list_of_results, + # ) + def info(self, limit: int = DEFAULT_LIMIT_INFO) -> ListOfNamespaceInfo: """ Get list of top n namespaces in the database + ! Warning: this function counts number of all projects in namespaces. + ! it does not filter private projects (It was done for efficiency reasons) :param limit: limit of results (top namespace ) :return: number_of_namespaces: int @@ -183,36 +228,24 @@ def info(self, limit: int = DEFAULT_LIMIT_INFO) -> ListOfNamespaceInfo: number_of_projects: int } """ - total_number_of_namespaces = self._count_namespace() - - statement = ( - select( - func.count(Projects.namespace).label("number_of_projects"), - Projects.namespace, - ) - .select_from(Projects) - .where(Projects.private.is_(False)) - .limit(limit) - .order_by(text("number_of_projects desc")) - .group_by(Projects.namespace) - ) - with Session(self._sa_engine) as session: - query_results = session.execute(statement).all() + results = session.scalars( + select(User).limit(limit).order_by(User.number_of_projects.desc()) + ) - list_of_results = [] - for result in query_results: - list_of_results.append( - NamespaceInfo( - namespace=result.namespace, - number_of_projects=result.number_of_projects, + list_of_results = [] + for result in results: + list_of_results.append( + NamespaceInfo( + namespace=result.namespace, + number_of_projects=result.number_of_projects, + ) ) + return ListOfNamespaceInfo( + number_of_namespaces=len(list_of_results), + limit=limit, + results=list_of_results, ) - return ListOfNamespaceInfo( - number_of_namespaces=total_number_of_namespaces, - limit=limit, - results=list_of_results, - ) def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: """ diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index afe5f20..1f07a3c 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1,39 +1,33 @@ import datetime import json import logging -from typing import Union, List, NoReturn, Mapping +from typing import Dict, List, NoReturn, Union -import peppy -from sqlalchemy import and_, delete, select -from sqlalchemy.exc import IntegrityError, NoResultFound -from sqlalchemy.orm import Session -from sqlalchemy import Select import numpy as np - +import peppy from peppy.const import ( - SAMPLE_RAW_DICT_KEY, - SUBSAMPLE_RAW_LIST_KEY, CONFIG_KEY, - SAMPLE_TABLE_INDEX_KEY, SAMPLE_NAME_ATTR, + SAMPLE_RAW_DICT_KEY, + SAMPLE_TABLE_INDEX_KEY, + SUBSAMPLE_RAW_LIST_KEY, ) +from sqlalchemy import Select, and_, delete, select +from sqlalchemy.exc import IntegrityError, NoResultFound +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import ( - DEFAULT_TAG, - DESCRIPTION_KEY, - NAME_KEY, - PKG_NAME, -) - -from pepdbagent.db_utils import Projects, Samples, Subsamples, BaseEngine +from pepdbagent.const import DEFAULT_TAG, DESCRIPTION_KEY, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME +from pepdbagent.db_utils import BaseEngine, Projects, Samples, Subsamples, User from pepdbagent.exceptions import ( + PEPDatabaseAgentError, + ProjectDuplicatedSampleGUIDsError, ProjectNotFoundError, ProjectUniqueNameError, - PEPDatabaseAgentError, + SampleTableUpdateError, ) -from pepdbagent.models import UpdateItems, UpdateModel, ProjectDict -from pepdbagent.utils import create_digest, registry_path_converter - +from pepdbagent.models import ProjectDict, UpdateItems, UpdateModel +from pepdbagent.utils import create_digest, generate_guid, order_samples, registry_path_converter _LOGGER = logging.getLogger(PKG_NAME) @@ -57,7 +51,8 @@ def get( namespace: str, name: str, tag: str = DEFAULT_TAG, - raw: bool = False, + raw: bool = True, + with_id: bool = False, ) -> Union[peppy.Project, dict, None]: """ Retrieve project from database by specifying namespace, name and tag @@ -66,6 +61,7 @@ def get( :param name: name of the project (Default: name is taken from the project object) :param tag: tag (or version) of the project. :param raw: retrieve unprocessed (raw) PEP dict. + :param with_id: retrieve project with id [default: False] :return: peppy.Project object with found project or dict with unprocessed PEP elements: { name: str @@ -97,24 +93,20 @@ def get( else: subsample_list = [] - # samples - samples_dict = { - sample_sa.row_number: sample_sa.sample - for sample_sa in found_prj.samples_mapping - } + sample_list = self._get_samples( + session=session, prj_id=found_prj.id, with_id=with_id + ) project_value = { CONFIG_KEY: found_prj.config, - SAMPLE_RAW_DICT_KEY: [samples_dict[key] for key in sorted(samples_dict)], + SAMPLE_RAW_DICT_KEY: sample_list, SUBSAMPLE_RAW_LIST_KEY: subsample_list, } - # project_value = found_prj.project_value - is_private = found_prj.private + if raw: return project_value else: project_obj = peppy.Project().from_dict(project_value) - project_obj.is_private = is_private return project_obj else: @@ -126,14 +118,42 @@ def get( except NoResultFound: raise ProjectNotFoundError + def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dict]: + """ + Get samples from the project. This method is used to retrieve samples from the project, + with open session object. + + :param session: open session object + :param prj_id: project id + :param with_id: retrieve sample with id + """ + samples_results = session.scalars(select(Samples).where(Samples.project_id == prj_id)) + result_dict = {} + for sample in samples_results: + sample_dict = sample.sample + if with_id: + sample_dict[PEPHUB_SAMPLE_ID_KEY] = sample.guid + + result_dict[sample.guid] = { + "sample": sample_dict, + "guid": sample.guid, + "parent_guid": sample.parent_guid, + } + + result_dict = order_samples(result_dict) + + ordered_samples_list = [sample["sample"] for sample in result_dict] + return ordered_samples_list + @staticmethod def _create_select_statement(name: str, namespace: str, tag: str = DEFAULT_TAG) -> Select: """ + Create simple select statement for retrieving project from database - :param name: - :param namespace: - :param tag: - :return: + :param name: name of the project + :param namespace: namespace of the project + :param tag: tag of the project + :return: select statement """ statement = select(Projects) statement = statement.where( @@ -188,8 +208,9 @@ def delete( raise ProjectNotFoundError( f"Can't delete unexciting project: '{namespace}/{name}:{tag}'." ) - with self._sa_engine.begin() as conn: - conn.execute( + + with Session(self._sa_engine) as session: + session.execute( delete(Projects).where( and_( Projects.namespace == namespace, @@ -199,7 +220,11 @@ def delete( ) ) - _LOGGER.info(f"Project '{namespace}/{name}:{tag} was successfully deleted'") + statement = select(User).where(User.namespace == namespace) + user = session.scalar(statement) + if user: + user.number_of_projects -= 1 + session.commit() def delete_by_rp( self, @@ -332,6 +357,15 @@ def create( self._add_subsamples_to_project(new_prj, subsamples) with Session(self._sa_engine) as session: + user = session.scalar(select(User).where(User.namespace == namespace)) + + if not user: + user = User(namespace=namespace) + session.add(user) + session.commit() + + user.number_of_projects += 1 + session.add(new_prj) session.commit() @@ -457,6 +491,13 @@ def update( is_private: Optional[bool] tag: Optional[str] name: Optional[str] + description: Optional[str] + is_private: Optional[bool] + pep_schema: Optional[str] + config: Optional[dict] + samples: Optional[List[dict]] + subsamples: Optional[List[List[dict]]] + pop: Optional[bool] } :param namespace: project namespace :param name: project name @@ -482,196 +523,146 @@ def update( statement = self._create_select_statement(name, namespace, tag) with Session(self._sa_engine) as session: - found_prj = session.scalar(statement) + found_prj: Projects = session.scalar(statement) - if found_prj: - _LOGGER.debug( - f"Project has been found: {found_prj.namespace}, {found_prj.name}" + if not found_prj: + raise ProjectNotFoundError( + f"Pep {namespace}/{name}:{tag} was not found. No items will be updated!" ) - for k, v in update_values.items(): - if getattr(found_prj, k) != v: - setattr(found_prj, k, v) - - # standardizing project name - if k == NAME_KEY: - if "config" in update_values: - update_values["config"][NAME_KEY] = v - else: - found_prj.config[NAME_KEY] = v - found_prj.name = found_prj.config[NAME_KEY] - - if "samples" in update_dict: - self._update_samples( - namespace=namespace, - name=name, - tag=tag, - samples_list=update_dict["samples"], - sample_name_key=update_dict["config"].get( - SAMPLE_TABLE_INDEX_KEY, "sample_name" - ), + for k, v in update_values.items(): + if getattr(found_prj, k) != v: + setattr(found_prj, k, v) + + # standardizing project name + if k == NAME_KEY: + if "config" in update_values: + update_values["config"][NAME_KEY] = v + else: + found_prj.config[NAME_KEY] = v + flag_modified(found_prj, "config") + found_prj.name = found_prj.config[NAME_KEY] + + if k == DESCRIPTION_KEY: + if "config" in update_values: + update_values["config"][DESCRIPTION_KEY] = v + else: + found_prj.config[DESCRIPTION_KEY] = v + # This line needed due to: https://github.com/sqlalchemy/sqlalchemy/issues/5218 + flag_modified(found_prj, "config") + + if "samples" in update_dict: + + if PEPHUB_SAMPLE_ID_KEY not in update_dict["samples"][0]: + raise SampleTableUpdateError( + f"pephub_sample_id '{PEPHUB_SAMPLE_ID_KEY}' is missing in samples." + f"Please provide it to update samples, or use overwrite method." ) - # if found_prj.samples_mapping: - # for sample in found_prj.samples_mapping: - # _LOGGER.debug(f"deleting samples: {str(sample)}") - # session.delete(sample) - # - # self._add_samples_to_project( - # found_prj, - # update_dict["samples"], - # sample_table_index=update_dict["config"].get(SAMPLE_TABLE_INDEX_KEY), - # ) - - if "subsamples" in update_dict: - if found_prj.subsamples_mapping: - for subsample in found_prj.subsamples_mapping: - _LOGGER.debug(f"deleting subsamples: {str(subsample)}") - session.delete(subsample) - - # Adding new subsamples - if update_dict["subsamples"]: - self._add_subsamples_to_project(found_prj, update_dict["subsamples"]) - found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + self._update_samples( + project_id=found_prj.id, + samples_list=update_dict["samples"], + sample_name_key=update_dict["config"].get( + SAMPLE_TABLE_INDEX_KEY, "sample_name" + ), + ) - session.commit() + if "subsamples" in update_dict: + if found_prj.subsamples_mapping: + for subsample in found_prj.subsamples_mapping: + _LOGGER.debug(f"deleting subsamples: {str(subsample)}") + session.delete(subsample) + + # Adding new subsamples + if update_dict["subsamples"]: + self._add_subsamples_to_project(found_prj, update_dict["subsamples"]) + + found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + + session.commit() return None else: raise ProjectNotFoundError("No items will be updated!") - @staticmethod - def _find_duplicates(sample_name_list: List[str]) -> List[str]: - seen = set() - duplicates = set() - for name in sample_name_list: - if name in seen: - duplicates.add(name) - else: - seen.add(name) - return list(duplicates) - def _update_samples( self, - namespace: str, - name: str, - tag: str, - samples_list: List[Mapping], + project_id: int, + samples_list: List[Dict[str, str]], sample_name_key: str = "sample_name", ) -> None: """ Update samples in the project - This is a new method that instead of deleting all samples and adding new ones, - updates samples and adds new ones if they don't exist + This is linked list method, that first finds differences in old and new samples list + and then updates, adds, inserts, deletes, or changes the order. + :param project_id: project id in PEPhub database :param samples_list: list of samples to be updated :param sample_name_key: key of the sample name :return: None """ - # TODO: This function is not ideal and is really slow. We should brainstorm this implementation - new_sample_names = [sample[sample_name_key] for sample in samples_list] with Session(self._sa_engine) as session: - project = session.scalar( - select(Projects).where( - and_( - Projects.namespace == namespace, Projects.name == name, Projects.tag == tag - ) + old_samples = session.scalars(select(Samples).where(Samples.project_id == project_id)) + + old_samples_mapping: dict = {sample.guid: sample for sample in old_samples} + old_samples_ids_set: set = set(old_samples_mapping.keys()) + new_samples_ids_list: list = [ + new_sample[PEPHUB_SAMPLE_ID_KEY] + for new_sample in samples_list + if new_sample[PEPHUB_SAMPLE_ID_KEY] != "" + and new_sample[PEPHUB_SAMPLE_ID_KEY] is not None + ] + new_samples_ids_set: set = set(new_samples_ids_list) + new_samples_dict: dict = { + new_sample[PEPHUB_SAMPLE_ID_KEY] or generate_guid(): new_sample + for new_sample in samples_list + } + + if len(new_samples_ids_list) != len(new_samples_ids_set): + raise ProjectDuplicatedSampleGUIDsError( + f"Samples have to have unique pephub_sample_id: '{PEPHUB_SAMPLE_ID_KEY}'." + f"If ids are duplicated, overwrite the project." ) - ) - old_sample_names = [sample.sample_name for sample in project.samples_mapping] - - # delete samples that are not in the new list - sample_names_copy = new_sample_names.copy() - for old_sample in old_sample_names: - if old_sample not in sample_names_copy: - this_sample = session.scalars( - select(Samples).where( - and_( - Samples.sample_name == old_sample, Samples.project_id == project.id - ) - ) + + # Check if something was deleted: + deleted_ids = old_samples_ids_set - new_samples_ids_set + + del new_samples_ids_list, new_samples_ids_set + + parent_id = None + parent_mapping = None + + # Main loop to update samples + for current_id, sample_value in new_samples_dict.items(): + new_sample = None + del sample_value[PEPHUB_SAMPLE_ID_KEY] + + if current_id not in old_samples_ids_set: + new_sample = Samples( + sample=sample_value, + guid=current_id, + sample_name=sample_value[sample_name_key], + row_number=0, + project_id=project_id, + parent_mapping=parent_mapping, ) - delete_samples_list = [k for k in this_sample] - session.delete(delete_samples_list[-1]) - else: - sample_names_copy.remove(old_sample) - - # update or add samples - order_number = 0 - added_sample_list = [] - for new_sample in samples_list: - order_number += 1 - - if new_sample[sample_name_key] not in added_sample_list: - added_sample_list.append(new_sample[sample_name_key]) - - if new_sample[sample_name_key] not in old_sample_names: - project.samples_mapping.append( - Samples( - sample=new_sample, - sample_name=new_sample[sample_name_key], - row_number=order_number, - ) - ) - else: - sample_mapping = session.scalar( - select(Samples).where( - and_( - Samples.sample_name == new_sample[sample_name_key], - Samples.project_id == project.id, - ) - ) - ) - sample_mapping.sample = new_sample - sample_mapping.row_number = order_number + session.add(new_sample) + else: - # if sample_name is duplicated is sample table, find second sample and update or add it. - if new_sample[sample_name_key] in old_sample_names: - sample_mappings = session.scalars( - select(Samples).where( - and_( - Samples.sample_name == new_sample[sample_name_key], - Samples.project_id == project.id, - ) - ) - ) - sample_mappings = [sample_mapping for sample_mapping in sample_mappings] - if len(sample_mappings) <= 1: - project.samples_mapping.append( - Samples( - sample=new_sample, - sample_name=new_sample[sample_name_key], - row_number=order_number, - ) - ) - else: - try: - sample_mapping = sample_mappings[ - added_sample_list.count(new_sample[sample_name_key]) - ] - sample_mapping.sample = new_sample - sample_mapping.row_number = order_number - - except Exception: - project.samples_mapping.append( - Samples( - sample=new_sample, - sample_name=new_sample[sample_name_key], - row_number=order_number, - ) - ) - added_sample_list.append(new_sample[sample_name_key]) - else: - project.samples_mapping.append( - Samples( - sample=new_sample, - sample_name=new_sample[sample_name_key], - row_number=order_number, - ) - ) - added_sample_list.append(new_sample[sample_name_key]) + if old_samples_mapping[current_id].sample != sample_value: + old_samples_mapping[current_id].sample = sample_value + old_samples_mapping[current_id].sample_name = sample_value[sample_name_key] + + if old_samples_mapping[current_id].parent_guid != parent_id: + old_samples_mapping[current_id].parent_mapping = parent_mapping + + parent_id = current_id + parent_mapping = new_sample or old_samples_mapping[current_id] + + for remove_id in deleted_ids: + session.delete(old_samples_mapping[remove_id]) session.commit() @@ -789,14 +780,18 @@ def _add_samples_to_project( :param sample_table_index: index of the sample table :return: NoReturn """ + previous_sample_guid = None for row_number, sample in enumerate(samples): - projects_sa.samples_mapping.append( - Samples( - sample=sample, - row_number=row_number, - sample_name=sample.get(sample_table_index), - ) + + sample = Samples( + sample=sample, + row_number=row_number, + sample_name=sample.get(sample_table_index), + parent_guid=previous_sample_guid, + guid=generate_guid(), ) + projects_sa.samples_mapping.append(sample) + previous_sample_guid = sample.guid return None @@ -806,6 +801,7 @@ def _add_subsamples_to_project( ) -> NoReturn: """ Add subsamples to the project sa object. (With commit this samples will be added to the 'subsamples table') + :param projects_sa: Projects sa object, in open session :param subsamples: list of subsamles to be added to the database :return: NoReturn @@ -845,7 +841,7 @@ def fork( fork_tag: str = None, description: str = None, private: bool = False, - ): + ) -> None: """ Fork project from one namespace to another @@ -859,6 +855,7 @@ def fork( :param private: boolean value if the project should be visible just for user that creates it. :return: None """ + self.create( project=self.get( namespace=original_namespace, @@ -873,19 +870,24 @@ def fork( is_private=private, ) original_statement = select(Projects).where( - Projects.namespace == original_namespace, - Projects.name == original_name, - Projects.tag == original_tag, + and_( + Projects.namespace == original_namespace, + Projects.name == original_name, + Projects.tag == original_tag, + ) ) fork_statement = select(Projects).where( - Projects.namespace == fork_namespace, - Projects.name == fork_name, - Projects.tag == fork_tag, + and_( + Projects.namespace == fork_namespace, + Projects.name == fork_name, + Projects.tag == fork_tag, + ) ) with Session(self._sa_engine) as session: original_prj = session.scalar(original_statement) fork_prj = session.scalar(fork_statement) + fork_prj.forked_from_id = original_prj.id fork_prj.pop = original_prj.pop fork_prj.submission_date = original_prj.submission_date @@ -893,7 +895,6 @@ def fork( fork_prj.description = description or original_prj.description session.commit() - return None def get_config(self, namespace: str, name: str, tag: str) -> Union[dict, None]: """ @@ -946,7 +947,9 @@ def get_subsamples(self, namespace: str, name: str, tag: str) -> Union[list, Non f"Did you supply a valid namespace and project?" ) - def get_samples(self, namespace: str, name: str, tag: str, raw: bool = True) -> list: + def get_samples( + self, namespace: str, name: str, tag: str, raw: bool = True, with_ids: bool = False + ) -> list: """ Get project samples by providing namespace, name, and tag @@ -954,15 +957,16 @@ def get_samples(self, namespace: str, name: str, tag: str, raw: bool = True) -> :param name: project name :param tag: project tag :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] + :param with_ids: if True, retrieve samples with ids. [Default: False] :return: list with project samples """ if raw: - return self.get(namespace=namespace, name=name, tag=tag, raw=True).get( - SAMPLE_RAW_DICT_KEY - ) + return self.get( + namespace=namespace, name=name, tag=tag, raw=True, with_id=with_ids + ).get(SAMPLE_RAW_DICT_KEY) return ( - self.get(namespace=namespace, name=name, tag=tag, raw=False) + self.get(namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids) .sample_table.replace({np.nan: None}) .to_dict(orient="records") ) diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index 7154ebe..7e8d89a 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -1,21 +1,17 @@ +import datetime import logging from typing import Union -import datetime import peppy from peppy.const import SAMPLE_TABLE_INDEX_KEY -from sqlalchemy import select, and_, func +from sqlalchemy import and_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified - -from pepdbagent.const import ( - DEFAULT_TAG, - PKG_NAME, -) -from pepdbagent.exceptions import SampleNotFoundError, SampleAlreadyExistsError - -from pepdbagent.db_utils import BaseEngine, Samples, Projects +from pepdbagent.const import DEFAULT_TAG, PKG_NAME +from pepdbagent.db_utils import BaseEngine, Projects, Samples +from pepdbagent.exceptions import SampleAlreadyExistsError, SampleNotFoundError +from pepdbagent.utils import generate_guid, order_samples _LOGGER = logging.getLogger(PKG_NAME) @@ -40,7 +36,7 @@ def get( name: str, sample_name: str, tag: str = DEFAULT_TAG, - raw: bool = False, + raw: bool = True, ) -> Union[peppy.Sample, dict, None]: """ Retrieve sample from the database using namespace, name, tag, and sample_name @@ -49,7 +45,7 @@ def get( :param name: name of the project (Default: name is taken from the project object) :param tag: tag (or version) of the project. :param sample_name: sample_name of the sample - :param raw: return raw dict or peppy.Sample object + :param raw: return raw dict or peppy.Sample object [Default: True] :return: peppy.Project object with found project or dict with unprocessed PEP elements: { name: str @@ -216,29 +212,10 @@ def add( raise KeyError( f"Sample index key {project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, 'sample_name')} not found in sample dict" ) - project_where_statement = ( - Samples.project_id - == select(Projects.id) - .where( - and_( - Projects.namespace == namespace, - Projects.name == name, - Projects.tag == tag, - ), - ) - .scalar_subquery() - ) statement = select(Samples).where( - and_(project_where_statement, Samples.sample_name == sample_name) + and_(Samples.project_id == project_mapping.id, Samples.sample_name == sample_name) ) - sample_mapping = session.scalar(statement) - row_number = ( - session.execute( - select(func.max(Samples.row_number)).where(project_where_statement) - ).one()[0] - or 0 - ) if sample_mapping and not overwrite: raise SampleAlreadyExistsError( @@ -257,9 +234,11 @@ def add( else: sample_mapping = Samples( sample=sample_dict, - row_number=row_number + 1, + row_number=0, project_id=project_mapping.id, sample_name=sample_name, + guid=generate_guid(), + parent_guid=self._get_last_sample_guid(project_mapping.id), ) project_mapping.number_of_samples += 1 project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) @@ -267,6 +246,28 @@ def add( session.add(sample_mapping) session.commit() + def _get_last_sample_guid(self, project_id: int) -> str: + """ + Get last sample guid from the project + + :param project_id: project_id of the project + :return: guid of the last sample + """ + statement = select(Samples).where(Samples.project_id == project_id) + with Session(self._sa_engine) as session: + samples_results = session.scalars(statement) + + result_dict = {} + for sample in samples_results: + sample_dict = sample.sample + + result_dict[sample.guid] = { + "sample": sample_dict, + "guid": sample.guid, + "parent_guid": sample.parent_guid, + } + return order_samples(result_dict)[-1]["guid"] + def delete( self, namespace: str, @@ -310,7 +311,11 @@ def delete( project_mapping = session.scalar(project_statement) if sample_mapping: + parent_mapping = sample_mapping.parent_mapping + child_mapping = sample_mapping.child_mapping session.delete(sample_mapping) + if parent_mapping: + child_mapping.parent_mapping = parent_mapping project_mapping.number_of_samples -= 1 project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) session.commit() diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 7ae4705..eea8f84 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -2,17 +2,13 @@ from typing import Union from sqlalchemy import and_, delete, select -from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session -from pepdbagent.const import ( - PKG_NAME, -) - -from pepdbagent.db_utils import BaseEngine, User, Stars, Projects +from pepdbagent.const import PKG_NAME +from pepdbagent.db_utils import BaseEngine, Projects, Stars, User +from pepdbagent.exceptions import ProjectAlreadyInFavorites, ProjectNotInFavorites from pepdbagent.models import AnnotationList, AnnotationModel -from pepdbagent.exceptions import ProjectNotInFavorites, ProjectAlreadyInFavorites - _LOGGER = logging.getLogger(PKG_NAME) diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index fb31a0f..8704c97 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -1,29 +1,24 @@ # View of the PEP. In other words, it is a part of the PEP, or subset of the samples in the PEP. import logging -from typing import Union, List +from typing import List, Union import peppy -from sqlalchemy import select, and_, delete -from sqlalchemy.orm import Session +from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session - -from pepdbagent.const import ( - DEFAULT_TAG, - PKG_NAME, -) +from pepdbagent.const import DEFAULT_TAG, PKG_NAME +from pepdbagent.db_utils import BaseEngine, Projects, Samples, Views, ViewSampleAssociation from pepdbagent.exceptions import ( - ViewNotFoundError, - SampleAlreadyInView, ProjectNotFoundError, + SampleAlreadyInView, SampleNotFoundError, - ViewAlreadyExistsError, SampleNotInViewError, + ViewAlreadyExistsError, + ViewNotFoundError, ) - -from pepdbagent.db_utils import BaseEngine, Samples, Projects, Views, ViewSampleAssociation -from pepdbagent.models import ViewAnnotation, CreateViewDictModel, ProjectViews +from pepdbagent.models import CreateViewDictModel, ProjectViews, ViewAnnotation _LOGGER = logging.getLogger(PKG_NAME) @@ -48,7 +43,7 @@ def get( name: str, tag: str = DEFAULT_TAG, view_name: str = None, - raw: bool = False, + raw: bool = True, ) -> Union[peppy.Project, dict, None]: """ Retrieve view of the project from the database. @@ -59,7 +54,7 @@ def get( :param name: name of the project (Default: name is taken from the project object) :param tag: tag of the project (Default: tag is taken from the project object) :param view_name: name of the view - :param raw: retrieve unprocessed (raw) PEP dict. + :param raw: retrieve unprocessed (raw) PEP dict. [Default: True] :return: peppy.Project object with found project or dict with unprocessed PEP elements: { name: str diff --git a/pepdbagent/pepdbagent.py b/pepdbagent/pepdbagent.py index 19995e5..1519c33 100644 --- a/pepdbagent/pepdbagent.py +++ b/pepdbagent/pepdbagent.py @@ -3,8 +3,8 @@ from pepdbagent.modules.annotation import PEPDatabaseAnnotation from pepdbagent.modules.namespace import PEPDatabaseNamespace from pepdbagent.modules.project import PEPDatabaseProject -from pepdbagent.modules.user import PEPDatabaseUser from pepdbagent.modules.sample import PEPDatabaseSample +from pepdbagent.modules.user import PEPDatabaseUser from pepdbagent.modules.view import PEPDatabaseView @@ -45,47 +45,48 @@ def __init__( ) sa_engine = pep_db_engine.engine - self.__sa_engine = sa_engine + self.pep_db_engine = pep_db_engine + self._sa_engine = sa_engine - self.__project = PEPDatabaseProject(pep_db_engine) - self.__annotation = PEPDatabaseAnnotation(pep_db_engine) - self.__namespace = PEPDatabaseNamespace(pep_db_engine) - self.__sample = PEPDatabaseSample(pep_db_engine) - self.__user = PEPDatabaseUser(pep_db_engine) - self.__view = PEPDatabaseView(pep_db_engine) + self._project = PEPDatabaseProject(pep_db_engine) + self._annotation = PEPDatabaseAnnotation(pep_db_engine) + self._namespace = PEPDatabaseNamespace(pep_db_engine) + self._sample = PEPDatabaseSample(pep_db_engine) + self._user = PEPDatabaseUser(pep_db_engine) + self._view = PEPDatabaseView(pep_db_engine) - self.__db_name = database + self._db_name = database @property def project(self) -> PEPDatabaseProject: - return self.__project + return self._project @property def annotation(self) -> PEPDatabaseAnnotation: - return self.__annotation + return self._annotation @property def namespace(self) -> PEPDatabaseNamespace: - return self.__namespace + return self._namespace @property def user(self) -> PEPDatabaseUser: - return self.__user + return self._user @property def sample(self) -> PEPDatabaseSample: - return self.__sample + return self._sample @property def view(self) -> PEPDatabaseView: - return self.__view + return self._view def __str__(self): return f"Connection to the database: '{self.__db_name}' is set!" def __exit__(self): - self.__sa_engine.__exit__() + self._sa_engine.__exit__() @property def connection(self): - return self.__sa_engine + return self._sa_engine diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index c58cff4..cbc596b 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -1,13 +1,14 @@ import datetime import json +import uuid from collections.abc import Iterable from hashlib import md5 -from typing import Tuple, Union +from typing import List, Tuple, Union import ubiquerg from peppy.const import SAMPLE_RAW_DICT_KEY -from .exceptions import RegistryPathError +from pepdbagent.exceptions import RegistryPathError def is_valid_registry_path(rpath: str) -> bool: @@ -100,7 +101,59 @@ def convert_date_string_to_date(date_string: str) -> datetime.datetime: """ Convert string into datetime format - :param date_str: date string in format [YYYY/MM/DD]. e.g. 2022/02/22 + :param date_string: date string in format [YYYY/MM/DD]. e.g. 2022/02/22 :return: datetime format """ return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta(days=1) + + +def order_samples(results: dict) -> List[dict]: + """ + Order samples by their parent_guid + + # TODO: To make this function more efficient, we should write it in Rust! + + :param results: dictionary of samples. Structure: { + "sample": sample_dict, + "guid": sample.guid, + "parent_guid": sample.parent_guid, + } + + :return: ordered list of samples + """ + # Find the Root Node + # Create a lookup dictionary for nodes by their GUIDs + guid_lookup = {entry["guid"]: entry for entry in results.values()} + + # Create a dictionary to map each GUID to its child GUID + parent_to_child = { + entry["parent_guid"]: entry["guid"] + for entry in results.values() + if entry["parent_guid"] is not None + } + + # Find the root node + root = None + for guid, entry in results.items(): + if entry["parent_guid"] is None: + root = entry + break + + if root is None: + raise ValueError("No root node found") + + ordered_sequence = [] + current = root + + while current is not None: + ordered_sequence.append(current) + current_guid = current["guid"] + if current_guid in parent_to_child: + current = guid_lookup[parent_to_child[current_guid]] + else: + current = None + return ordered_sequence + + +def generate_guid() -> str: + return str(uuid.uuid4()) diff --git a/tests/README.md b/tests/README.md index 24cceec..0d655d8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,9 +5,9 @@ ```txt docker run --rm -it --name pep-db \ -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=docker \ + -e POSTGRES_PASSWORD=pass8743hf9h23f87h437 \ -e POSTGRES_DB=pep-db \ - -p 5432:5432 postgres + -p 127.0.0.1:5432:5432 postgres ``` diff --git a/tests/conftest.py b/tests/conftest.py index 32d3dbe..d1a1c6b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,84 +1,83 @@ -import peppy -import pytest -import os - -from sqlalchemy import create_engine -from sqlalchemy import text - -from pepdbagent import PEPDatabaseAgent - -DNS = "postgresql+psycopg://postgres:docker@localhost:5432/pep-db" - - -DATA_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "tests", - "data", -) - - -def get_path_to_example_file(namespace, project_name): - return os.path.join(DATA_PATH, namespace, project_name, "project_config.yaml") - - -@pytest.fixture -def list_of_available_peps(): - pep_namespaces = os.listdir(DATA_PATH) - projects = {} - for np in pep_namespaces: - pep_name = os.listdir(os.path.join(DATA_PATH, np)) - projects[np] = {p: get_path_to_example_file(np, p) for p in pep_name} - return projects - - -@pytest.fixture(scope="function") -def initiate_pepdb_con( - list_of_available_peps, -): - sa_engine = create_engine(DNS) - with sa_engine.begin() as conn: - conn.execute(text("DROP table IF EXISTS projects CASCADE")) - conn.execute(text("DROP table IF EXISTS samples CASCADE")) - conn.execute(text("DROP table IF EXISTS subsamples CASCADE")) - conn.execute(text("DROP table IF EXISTS stars CASCADE")) - conn.execute(text("DROP table IF EXISTS users CASCADE")) - conn.execute(text("DROP table IF EXISTS views CASCADE")) - conn.execute(text("DROP table IF EXISTS views_samples CASCADE")) - - pepdb_con = PEPDatabaseAgent(dsn=DNS, echo=True) - for namespace, item in list_of_available_peps.items(): - if namespace == "private_test": - private = True - else: - private = False - for name, path in item.items(): - prj = peppy.Project(path) - pepdb_con.project.create( - namespace=namespace, - name=name, - tag="default", - is_private=private, - project=prj, - overwrite=True, - pep_schema="random_schema_name", - ) - - yield pepdb_con - - -@pytest.fixture(scope="function") -def initiate_empty_pepdb_con( - list_of_available_peps, -): - """ - create connection without adding peps to the db - """ - # sa_engine = create_engine(DNS) - # with sa_engine.begin() as conn: - # conn.execute(text("DROP table IF EXISTS projects CASCADE")) - # conn.execute(text("DROP table IF EXISTS samples CASCADE")) - # conn.execute(text("DROP table IF EXISTS subsamples CASCADE")) - - pepdb_con = PEPDatabaseAgent(dsn=DNS, echo=False) - - yield pepdb_con +# import os +# +# import peppy +# import pytest +# from sqlalchemy import create_engine, text +# +# from pepdbagent import PEPDatabaseAgent +# +# DNS = "postgresql+psycopg://postgres:docker@localhost:5432/pep-db" +# +# +# DATA_PATH = os.path.join( +# os.path.dirname(os.path.dirname(os.path.abspath(__file__))), +# "tests", +# "data", +# ) +# +# +# def get_path_to_example_file(namespace, project_name): +# return os.path.join(DATA_PATH, namespace, project_name, "project_config.yaml") +# +# +# @pytest.fixture +# def list_of_available_peps(): +# pep_namespaces = os.listdir(DATA_PATH) +# projects = {} +# for np in pep_namespaces: +# pep_name = os.listdir(os.path.join(DATA_PATH, np)) +# projects[np] = {p: get_path_to_example_file(np, p) for p in pep_name} +# return projects +# +# +# @pytest.fixture(scope="function") +# def initiate_pepdb_con( +# list_of_available_peps, +# ): +# sa_engine = create_engine(DNS) +# with sa_engine.begin() as conn: +# conn.execute(text("DROP table IF EXISTS projects CASCADE")) +# conn.execute(text("DROP table IF EXISTS samples CASCADE")) +# conn.execute(text("DROP table IF EXISTS subsamples CASCADE")) +# conn.execute(text("DROP table IF EXISTS stars CASCADE")) +# conn.execute(text("DROP table IF EXISTS users CASCADE")) +# conn.execute(text("DROP table IF EXISTS views CASCADE")) +# conn.execute(text("DROP table IF EXISTS views_samples CASCADE")) +# +# pepdb_con = PEPDatabaseAgent(dsn=DNS, echo=True) +# for namespace, item in list_of_available_peps.items(): +# if namespace == "private_test": +# private = True +# else: +# private = False +# for name, path in item.items(): +# prj = peppy.Project(path) +# pepdb_con.project.create( +# namespace=namespace, +# name=name, +# tag="default", +# is_private=private, +# project=prj, +# overwrite=True, +# pep_schema="random_schema_name", +# ) +# +# yield pepdb_con +# +# +# @pytest.fixture(scope="function") +# def initiate_empty_pepdb_con( +# list_of_available_peps, +# ): +# """ +# create connection without adding peps to the db +# """ +# # sa_engine = create_engine(DNS) +# # with sa_engine.begin() as conn: +# # conn.execute(text("DROP table IF EXISTS projects CASCADE")) +# # conn.execute(text("DROP table IF EXISTS samples CASCADE")) +# # conn.execute(text("DROP table IF EXISTS subsamples CASCADE")) +# +# pepdb_con = PEPDatabaseAgent(dsn=DNS, echo=False) +# +# yield pepdb_con diff --git a/tests/test_annotation.py b/tests/test_annotation.py new file mode 100644 index 0000000..e6778e3 --- /dev/null +++ b/tests/test_annotation.py @@ -0,0 +1,323 @@ +import datetime +import pytest +from pepdbagent.exceptions import FilterError, ProjectNotFoundError + +from .utils import PEPDBAgentContextManager + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestAnnotation: + """ + Test function within annotation class + """ + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ["namespace2", "derive"], + ["namespace2", "imply"], + ["namespace3", "subtable1"], + ], + ) + def test_annotation_of_one_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get( + namespace=namespace, + name=name, + tag="default", + ) + assert result.results[0].namespace == namespace + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace6", "amendments1"], + ], + ) + def test_annotation_of_one_non_existing_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + with pytest.raises(ProjectNotFoundError): + agent.annotation.get( + namespace=namespace, + name=name, + tag="default", + ) + + @pytest.mark.parametrize( + "namespace, n_projects", + [ + ["namespace1", 6], + ["namespace2", 8], + ["namespace3", 10], + ["private", 0], + ["private_test", 0], + ], + ) + def test_annotation_all(self, namespace, n_projects): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get( + namespace=namespace, + ) + assert result.count == n_projects + assert len(result.results) == n_projects + + @pytest.mark.parametrize( + "namespace, n_projects", + [ + ["namespace1", 6], + ["namespace2", 8], + ["namespace3", 10], + ["private", 0], + ["private_test", 6], + ], + ) + @pytest.mark.parametrize("admin", ("private_test", ["private_test", "bbb"])) + def test_annotation_all_private(self, namespace, n_projects, admin): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get(namespace=namespace, admin=admin) + assert result.count == n_projects + assert len(result.results) == n_projects + + @pytest.mark.parametrize( + "namespace, limit, n_projects", + [ + ["namespace1", 3, 6], + ["namespace2", 2, 8], + ["namespace3", 8, 10], + ["private", 0, 0], + ["private_test", 5, 6], + ], + ) + @pytest.mark.parametrize("admin", ("private_test", ["private_test", "bbb"])) + def test_annotation_limit(self, namespace, limit, admin, n_projects): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get(namespace=namespace, admin=admin, limit=limit) + assert result.count == n_projects + assert len(result.results) == limit + + @pytest.mark.parametrize( + "namespace, order_by, first_name", + [ + ["namespace1", "name", "amendments1"], + ["namespace2", "name", "biocproject_exceptions"], + ["namespace3", "name", "node_alias"], + ["private_test", "name", "amendments1"], + ], + ) + @pytest.mark.parametrize("admin", ["private_test"]) + def test_order_by(self, namespace, admin, order_by, first_name): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get(namespace=namespace, admin=admin, order_by=order_by) + assert result.results[0].name == first_name + + @pytest.mark.parametrize( + "namespace, order_by, last_name", + [ + ["namespace1", "name", "biocproject"], + ["namespace2", "name", "imports"], + ["namespace3", "name", "subtables"], + ["private_test", "name", "subtable3"], + ], + ) + @pytest.mark.parametrize("admin", ["private_test"]) + def test_order_by_desc(self, namespace, admin, order_by, last_name): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get( + namespace=namespace, + admin=admin, + order_by=order_by, + order_desc=True, + ) + assert result.results[0].name == last_name + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ["namespace2", "proj", 2], + ["namespace3", "ABLE", 6], + ["private_test", "a", 0], + [None, "re", 2], + ], + ) + def test_name_search(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get(namespace=namespace, query=query) + assert len(result.results) == found_number + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ["namespace2", "proj", 2], + ["namespace3", "ABLE", 6], + ["private_test", "b", 2], + [None, "re", 3], + ], + ) + def test_name_search_private(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get(namespace=namespace, query=query, admin="private_test") + assert len(result.results) == found_number + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ["namespace2", "derive"], + ["namespace2", "imply"], + ["namespace3", "subtable1"], + ], + ) + def test_all_annotations_are_returned(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get( + namespace=namespace, + name=name, + tag="default", + ) + assert result.results[0].model_fields_set == { + "is_private", + "tag", + "namespace", + "digest", + "description", + "number_of_samples", + "name", + "last_update_date", + "submission_date", + "pep_schema", + "pop", + "stars_number", + "forked_from", + } + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + [None, "re", 3], + ], + ) + def test_search_filter_success(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + date_now = datetime.datetime.now() + datetime.timedelta(days=1) + date_old = datetime.datetime.now() - datetime.timedelta(days=5) + result = agent.annotation.get( + namespace=namespace, + query=query, + admin="private_test", + filter_by="submission_date", + filter_start_date=date_old.strftime("%Y/%m/%d"), + filter_end_date=date_now.strftime("%Y/%m/%d"), + ) + assert len(result.results) == found_number + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 0], + [None, "re", 0], + ], + ) + def test_search_filter_zero_prj(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + date_now = datetime.datetime.now() - datetime.timedelta(days=2) + date_old = date_now - datetime.timedelta(days=2) + result = agent.annotation.get( + namespace=namespace, + query=query, + admin="private_test", + filter_by="submission_date", + filter_start_date=date_old.strftime("%Y/%m/%d"), + filter_end_date=date_now.strftime("%Y/%m/%d"), + ) + assert len(result.results) == found_number + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ], + ) + def test_search_incorrect_filter_by_string(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + date_now = datetime.datetime.now() - datetime.timedelta(days=2) + date_old = date_now - datetime.timedelta(days=2) + with pytest.raises(FilterError): + agent.annotation.get( + namespace=namespace, + query=query, + admin="private_test", + filter_by="incorrect", + filter_start_date=date_old.strftime("%Y/%m/%d"), + filter_end_date=date_now.strftime("%Y/%m/%d"), + ) + + @pytest.mark.parametrize( + "rp_list, admin, found_number", + [ + [ + [ + "namespace1/amendments1:default", + "namespace1/amendments2:default", + "namespace2/derive:default", + "private_test/amendments1:default", + ], + "namespace1", + 4, + ], + [ + [ + "namespace1/amendments1:default", + "namespace1/amendments2:default", + "namespace2/derive:default", + "private_test/amendments1:default", + ], + "private_test", + 4, + ], + ], + ) + def test_get_annotation_by_rp_list(self, rp_list, admin, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + + result = agent.annotation.get_by_rp_list(rp_list) + assert len(result.results) == found_number + + def test_get_annotation_by_rp_enpty_list(self): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get_by_rp_list([]) + assert len(result.results) == 0 + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ], + ) + def test_search_incorrect_incorrect_pep_type(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + + with pytest.raises(ValueError): + agent.annotation.get(namespace=namespace, pep_type="incorrect") + + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ], + ) + def test_project_list_without_annotation(self, namespace, query, found_number): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get_projects_list( + namespace=namespace, + search_str=query, + ) + assert len(result) == found_number diff --git a/tests/test_namespace.py b/tests/test_namespace.py new file mode 100644 index 0000000..432a9c2 --- /dev/null +++ b/tests/test_namespace.py @@ -0,0 +1,157 @@ +import pytest +from pepdbagent.exceptions import ProjectAlreadyInFavorites, ProjectNotInFavorites + +from .utils import PEPDBAgentContextManager + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestNamespace: + """ + Test function within namespace class + """ + + def test_annotation(self): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.namespace.get() + assert len(result.results) == 3 + + def test_annotation_private(self): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.namespace.get(admin="private_test") + assert len(result.results) == 4 + + @pytest.mark.skip( + "Skipping test because we are not taking into account the private projects (We are counting all of them)" + ) + def test_namespace_info_private(self): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace="private_test", + name="derive", + tag="default", + update_dict={"is_private": False}, + ) + result = agent.namespace.info() + assert len(result.results) == 4 + assert result.results[3].number_of_projects == 1 + + def test_namespace_info_all(self): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace="private_test", + name="derive", + tag="default", + update_dict={"is_private": False}, + ) + result = agent.namespace.info() + assert len(result.results) == 4 + assert result.results[3].number_of_projects == 6 + + def test_namespace_stats(self): + with PEPDBAgentContextManager(add_data=True) as agent: + stat_result = agent.namespace.stats(monthly=True) + assert next(iter(stat_result.projects_created.values()), 0) == 30 + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestFavorites: + """ + Test function within user class + """ + + def test_add_projects_to_favorites(self): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.annotation.get( + namespace="namespace1", + ) + for project in result.results: + agent.user.add_project_to_favorites( + "random_namespace", project.namespace, project.name, "default" + ) + fav_results = agent.user.get_favorites("random_namespace") + + assert fav_results.count == len(result.results) + + # This can fail if the order of the results is different + assert fav_results.results[0].namespace == result.results[0].namespace + + def test_count_project_none(self): + with PEPDBAgentContextManager(add_data=True) as agent: + result = agent.user.get_favorites("private_test") + assert result.count == 0 + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_count_project_one(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.user.add_project_to_favorites(namespace, namespace, name, "default") + result = agent.user.get_favorites("namespace1") + assert result.count == 1 + result1 = agent.user.get_favorites("private_test") + assert result1.count == 0 + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_remove_from_favorite(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites("namespace1", namespace, "amendments2", "default") + result = agent.user.get_favorites("namespace1") + assert result.count == len(result.results) == 2 + agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + result = agent.user.get_favorites("namespace1") + assert result.count == len(result.results) == 1 + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_remove_from_favorite_error(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + with pytest.raises(ProjectNotInFavorites): + agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_favorites_duplication_error(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + with pytest.raises(ProjectAlreadyInFavorites): + agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_annotation_favorite_number(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + annotations_in_namespace = agent.annotation.get("namespace1") + + for prj_annot in annotations_in_namespace.results: + if prj_annot.name == name: + assert prj_annot.stars_number == 1 + else: + assert prj_annot.stars_number == 0 diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py deleted file mode 100644 index 7fd7d66..0000000 --- a/tests/test_pepagent.py +++ /dev/null @@ -1,1426 +0,0 @@ -import datetime -import os -import warnings - -import peppy -import pytest -from sqlalchemy.exc import OperationalError -import numpy as np - -import pepdbagent -from pepdbagent.exceptions import ( - FilterError, - ProjectNotFoundError, - ProjectNotInFavorites, - ProjectAlreadyInFavorites, - SampleNotFoundError, - ViewNotFoundError, - SampleAlreadyInView, - SampleNotInViewError, -) -from .conftest import DNS - -DATA_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "tests", - "data", -) - - -def get_path_to_example_file(namespace, project_name): - return os.path.join(DATA_PATH, namespace, project_name, "project_config.yaml") - - -def db_setup(): - # Check if the database is setup - try: - pepdbagent.PEPDatabaseAgent(dsn=DNS) - except OperationalError: - warnings.warn( - UserWarning( - f"Skipping tests, because DB is not setup. {DNS}. To setup DB go to README.md" - ) - ) - return False - return True - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestProject: - """ - Test project methods - """ - - def test_create_project(self, initiate_empty_pepdb_con, list_of_available_peps): - prj = peppy.Project(list_of_available_peps["namespace3"]["subtables"]) - initiate_empty_pepdb_con.project.create( - prj, namespace="test", name="imply", overwrite=True - ) - assert True - - def test_create_project_from_dict(self, initiate_empty_pepdb_con, list_of_available_peps): - prj = peppy.Project(list_of_available_peps["namespace3"]["subtables"]) - initiate_empty_pepdb_con.project.create( - prj.to_dict(extended=True, orient="records"), - namespace="test", - name="imply", - overwrite=True, - ) - assert True - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace1", "basic"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ["namespace3", "piface"], - ["namespace3", "subtable2"], - ], - ) - def test_get_project(self, initiate_pepdb_con, namespace, name): - kk = initiate_pepdb_con.project.get(namespace=namespace, name=name, tag="default") - ff = peppy.Project(get_path_to_example_file(namespace, name)) - assert kk == ff - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_get_config(self, initiate_pepdb_con, namespace, name): - description = "" - kk = initiate_pepdb_con.project.get_config( - namespace=namespace, - name=name, - tag="default", - ) - ff = peppy.Project(get_path_to_example_file(namespace, name)) - ff.description = description - ff.name = name - assert kk == ff.config - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace3", "subtables"], - ], - ) - def test_get_subsamples(self, initiate_pepdb_con, namespace, name): - prj_subtables = initiate_pepdb_con.project.get_subsamples( - namespace=namespace, - name=name, - tag="default", - ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) - - assert ( - prj_subtables - == orgiginal_prj.to_dict(extended=True, orient="records")["_subsample_list"] - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace3", "subtables"], - ], - ) - def test_get_samples_raw(self, initiate_pepdb_con, namespace, name): - prj_samples = initiate_pepdb_con.project.get_samples( - namespace=namespace, name=name, tag="default", raw=True - ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) - - assert ( - prj_samples == orgiginal_prj.to_dict(extended=True, orient="records")["_sample_dict"] - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace3", "subtables"], - ], - ) - def test_get_samples_processed(self, initiate_pepdb_con, namespace, name): - prj_samples = initiate_pepdb_con.project.get_samples( - namespace=namespace, - name=name, - tag="default", - raw=False, - ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) - - assert prj_samples == orgiginal_prj.sample_table.replace({np.nan: None}).to_dict( - orient="records" - ) - - @pytest.mark.parametrize( - "namespace, name,tag", - [ - ["incorrect_namespace", "amendments1", "default"], - ["namespace1", "subtable2", "default"], - ["namespace3", "basic", "default"], - ["namespace3", "subtable2", "incorrect_tag"], - ["namespace1", "incorrect_name", "default"], - ], - ) - def test_get_project_error(self, initiate_pepdb_con, namespace, name, tag): - with pytest.raises(ProjectNotFoundError, match="Project does not exist."): - initiate_pepdb_con.project.get(namespace=namespace, name=name, tag=tag) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ], - ) - def test_overwrite_project(self, initiate_pepdb_con, namespace, name): - new_prj = initiate_pepdb_con.project.get(namespace="namespace1", name="basic") - - initiate_pepdb_con.project.create( - project=new_prj, - namespace=namespace, - name=name, - tag="default", - overwrite=True, - ) - - assert initiate_pepdb_con.project.get(namespace=namespace, name=name) == new_prj - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ], - ) - def test_delete_project(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.project.delete(namespace=namespace, name=name, tag="default") - - with pytest.raises(ProjectNotFoundError, match="Project does not exist."): - initiate_pepdb_con.project.get(namespace=namespace, name=name, tag="default") - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ], - ) - def test_fork_projects(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.project.fork( - original_namespace=namespace, - original_name=name, - original_tag="default", - fork_namespace="new_namespace", - fork_name="new_name", - fork_tag="new_tag", - ) - - assert initiate_pepdb_con.project.exists( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - result = initiate_pepdb_con.annotation.get( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - assert result.results[0].forked_from == f"{namespace}/{name}:default" - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ], - ) - def test_parent_project_delete(self, initiate_pepdb_con, namespace, name): - """ - Test if parent project is deleted, forked project is not deleted - """ - initiate_pepdb_con.project.fork( - original_namespace=namespace, - original_name=name, - original_tag="default", - fork_namespace="new_namespace", - fork_name="new_name", - fork_tag="new_tag", - ) - - assert initiate_pepdb_con.project.exists( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - initiate_pepdb_con.project.delete(namespace=namespace, name=name, tag="default") - assert initiate_pepdb_con.project.exists( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ], - ) - def test_child_project_delete(self, initiate_pepdb_con, namespace, name): - """ - Test if child project is deleted, parent project is not deleted - """ - initiate_pepdb_con.project.fork( - original_namespace=namespace, - original_name=name, - original_tag="default", - fork_namespace="new_namespace", - fork_name="new_name", - fork_tag="new_tag", - ) - - assert initiate_pepdb_con.project.exists( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - assert initiate_pepdb_con.project.exists(namespace=namespace, name=name, tag="default") - initiate_pepdb_con.project.delete( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - assert initiate_pepdb_con.project.exists(namespace=namespace, name=name, tag="default") - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ], - ) - def test_project_can_be_forked_twice(self, initiate_pepdb_con, namespace, name): - """ - Test if project can be forked twice - """ - initiate_pepdb_con.project.fork( - original_namespace=namespace, - original_name=name, - original_tag="default", - fork_namespace="new_namespace", - fork_name="new_name", - fork_tag="new_tag", - ) - initiate_pepdb_con.project.fork( - original_namespace=namespace, - original_name=name, - original_tag="default", - fork_namespace="new_namespace2", - fork_name="new_name2", - fork_tag="new_tag2", - ) - - result = initiate_pepdb_con.annotation.get( - namespace="new_namespace", name="new_name", tag="new_tag" - ) - assert result.results[0].forked_from == f"{namespace}/{name}:default" - - result = initiate_pepdb_con.annotation.get( - namespace="new_namespace2", name="new_name2", tag="new_tag2" - ) - assert result.results[0].forked_from == f"{namespace}/{name}:default" - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestProjectUpdate: - @pytest.mark.parametrize( - "namespace, name,new_name", - [ - ["namespace1", "amendments1", "name1"], - ["namespace1", "amendments2", "name2"], - ["namespace2", "derive", "name3"], - ["namespace1", "basic", "name4"], - ["namespace2", "derive", "name5"], - ], - ) - def test_update_project_name(self, initiate_pepdb_con, namespace, name, new_name): - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"name": new_name}, - ) - assert initiate_pepdb_con.project.exists(namespace=namespace, name=new_name, tag="default") - - @pytest.mark.parametrize( - "namespace, name, new_tag", - [ - ["namespace1", "amendments1", "tag1"], - ["namespace1", "amendments2", "tag2"], - ["namespace2", "derive", "tag3"], - ["namespace1", "basic", "tag4"], - ["namespace2", "derive", "tag5"], - ], - ) - def test_update_project_tag(self, initiate_pepdb_con, namespace, name, new_tag): - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"tag": new_tag}, - ) - assert initiate_pepdb_con.project.exists(namespace=namespace, name=name, tag=new_tag) - - @pytest.mark.parametrize( - "namespace, name, new_description", - [ - ["namespace1", "amendments1", "desc1 f"], - ["namespace1", "amendments2", "desc2 f"], - ["namespace2", "derive", "desc3 f"], - ["namespace1", "basic", "desc4 f"], - ["namespace2", "derive", "desc5 f"], - ], - ) - def test_update_project_description( - self, initiate_pepdb_con, namespace, name, new_description - ): - prj = initiate_pepdb_con.project.get(namespace=namespace, name=name) - prj.description = new_description - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"project": prj}, - ) - - assert ( - initiate_pepdb_con.project.get(namespace=namespace, name=name).description - == new_description - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace3", "subtable1"], - # ["namespace2", "derive"], - # ["namespace2", "imply"], - ], - ) - def test_update_whole_project(self, initiate_pepdb_con, namespace, name): - new_prj = initiate_pepdb_con.project.get(namespace="namespace1", name="basic") - # update name. If name is different, it will update name too - new_prj.name = name - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"project": new_prj}, - ) - assert initiate_pepdb_con.project.get(namespace=namespace, name=name) == new_prj - - @pytest.mark.parametrize( - "namespace, name, pep_schema", - [ - ["namespace1", "amendments1", "schema1"], - ["namespace2", "derive", "schema3"], - ["namespace1", "basic", "schema4"], - ["namespace2", "derive", "schema5"], - ], - ) - def test_update_pep_schema(self, initiate_pepdb_con, namespace, name, pep_schema): - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"pep_schema": pep_schema}, - ) - res = initiate_pepdb_con.annotation.get(namespace, name, "default") - assert res.results[0].pep_schema == pep_schema - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_update_project_private(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"is_private": True}, - ) - - is_private = ( - initiate_pepdb_con.annotation.get( - namespace=namespace, name=name, tag="default", admin=[namespace] - ) - .results[0] - .is_private - ) - assert is_private is True - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_update_project_pop(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"pop": True}, - ) - - pop = ( - initiate_pepdb_con.annotation.get( - namespace=namespace, name=name, tag="default", admin=[namespace] - ) - .results[0] - .pop - ) - assert pop is True - - # Update to pop = False and check if it is updated - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"pop": False}, - ) - - pop = ( - initiate_pepdb_con.annotation.get( - namespace=namespace, name=name, tag="default", admin=[namespace] - ) - .results[0] - .pop - ) - assert pop is False - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "basic"], - ], - ) - def test_project_can_have_2_sample_names(self, initiate_pepdb_con, namespace, name): - """ - In PEP 2.1.0 project can have 2 rows with same sample name, - ensure that update works correctly - """ - new_prj = initiate_pepdb_con.project.get(namespace=namespace, name=name) - prj_dict = new_prj.to_dict(extended=True, orient="records") - - prj_dict["_sample_dict"].append( - {"file": "data/frog23_data.txt", "protocol": "anySample3Type", "sample_name": "frog_2"} - ) - prj_dict["_sample_dict"].append( - { - "file": "data/frog23_data.txt4", - "protocol": "anySample3Type4", - "sample_name": "frog_2", - } - ) - - new_prj.description = "new_description" - initiate_pepdb_con.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"project": peppy.Project.from_dict(prj_dict)}, - ) - - prj = initiate_pepdb_con.project.get(namespace=namespace, name=name, raw=True) - - assert len(prj["_sample_dict"]) == 4 - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestAnnotation: - """ - Test function within annotation class - """ - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ["namespace3", "subtable1"], - ], - ) - def test_annotation_of_one_project(self, initiate_pepdb_con, namespace, name): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - name=name, - tag="default", - ) - assert result.results[0].namespace == namespace - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace6", "amendments1"], - ], - ) - def test_annotation_of_one_non_existing_project(self, initiate_pepdb_con, namespace, name): - with pytest.raises(ProjectNotFoundError): - initiate_pepdb_con.annotation.get( - namespace=namespace, - name=name, - tag="default", - ) - - @pytest.mark.parametrize( - "namespace, n_projects", - [ - ["namespace1", 6], - ["namespace2", 8], - ["namespace3", 10], - ["private", 0], - ["private_test", 0], - ], - ) - def test_annotation_all(self, initiate_pepdb_con, namespace, n_projects): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - ) - assert result.count == n_projects - assert len(result.results) == n_projects - - @pytest.mark.parametrize( - "namespace, n_projects", - [ - ["namespace1", 6], - ["namespace2", 8], - ["namespace3", 10], - ["private", 0], - ["private_test", 6], - ], - ) - @pytest.mark.parametrize("admin", ("private_test", ["private_test", "bbb"])) - def test_annotation_all_private(self, initiate_pepdb_con, namespace, n_projects, admin): - result = initiate_pepdb_con.annotation.get(namespace=namespace, admin=admin) - assert result.count == n_projects - assert len(result.results) == n_projects - - @pytest.mark.parametrize( - "namespace, limit, n_projects", - [ - ["namespace1", 3, 6], - ["namespace2", 2, 8], - ["namespace3", 8, 10], - ["private", 0, 0], - ["private_test", 5, 6], - ], - ) - @pytest.mark.parametrize("admin", ("private_test", ["private_test", "bbb"])) - def test_annotation_limit(self, initiate_pepdb_con, namespace, limit, admin, n_projects): - result = initiate_pepdb_con.annotation.get(namespace=namespace, admin=admin, limit=limit) - assert result.count == n_projects - assert len(result.results) == limit - - @pytest.mark.parametrize( - "namespace, order_by, first_name", - [ - ["namespace1", "name", "amendments1"], - ["namespace2", "name", "biocproject_exceptions"], - ["namespace3", "name", "node_alias"], - ["private_test", "name", "amendments1"], - ], - ) - @pytest.mark.parametrize("admin", ["private_test"]) - def test_order_by(self, initiate_pepdb_con, namespace, admin, order_by, first_name): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, admin=admin, order_by=order_by - ) - assert result.results[0].name == first_name - - @pytest.mark.parametrize( - "namespace, order_by, last_name", - [ - ["namespace1", "name", "biocproject"], - ["namespace2", "name", "imports"], - ["namespace3", "name", "subtables"], - ["private_test", "name", "subtable3"], - ], - ) - @pytest.mark.parametrize("admin", ["private_test"]) - def test_order_by_desc(self, initiate_pepdb_con, namespace, admin, order_by, last_name): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - admin=admin, - order_by=order_by, - order_desc=True, - ) - assert result.results[0].name == last_name - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - ["namespace2", "proj", 2], - ["namespace3", "ABLE", 6], - ["private_test", "a", 0], - [None, "re", 2], - ], - ) - def test_name_search(self, initiate_pepdb_con, namespace, query, found_number): - result = initiate_pepdb_con.annotation.get(namespace=namespace, query=query) - assert len(result.results) == found_number - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - ["namespace2", "proj", 2], - ["namespace3", "ABLE", 6], - ["private_test", "b", 2], - [None, "re", 3], - ], - ) - def test_name_search_private(self, initiate_pepdb_con, namespace, query, found_number): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, query=query, admin="private_test" - ) - assert len(result.results) == found_number - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], - ["namespace3", "subtable1"], - ], - ) - def test_all_annotations_are_returned(self, initiate_pepdb_con, namespace, name): - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - name=name, - tag="default", - ) - assert result.results[0].model_fields_set == { - "is_private", - "tag", - "namespace", - "digest", - "description", - "number_of_samples", - "name", - "last_update_date", - "submission_date", - "pep_schema", - "pop", - "stars_number", - "forked_from", - } - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - [None, "re", 3], - ], - ) - def test_search_filter_success(self, initiate_pepdb_con, namespace, query, found_number): - date_now = datetime.datetime.now() + datetime.timedelta(days=1) - date_old = datetime.datetime.now() - datetime.timedelta(days=5) - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - query=query, - admin="private_test", - filter_by="submission_date", - filter_start_date=date_old.strftime("%Y/%m/%d"), - filter_end_date=date_now.strftime("%Y/%m/%d"), - ) - assert len(result.results) == found_number - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 0], - [None, "re", 0], - ], - ) - def test_search_filter_zero_prj(self, initiate_pepdb_con, namespace, query, found_number): - date_now = datetime.datetime.now() - datetime.timedelta(days=2) - date_old = date_now - datetime.timedelta(days=2) - result = initiate_pepdb_con.annotation.get( - namespace=namespace, - query=query, - admin="private_test", - filter_by="submission_date", - filter_start_date=date_old.strftime("%Y/%m/%d"), - filter_end_date=date_now.strftime("%Y/%m/%d"), - ) - assert len(result.results) == found_number - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - ], - ) - def test_search_incorrect_filter_by_string( - self, initiate_pepdb_con, namespace, query, found_number - ): - date_now = datetime.datetime.now() - datetime.timedelta(days=2) - date_old = date_now - datetime.timedelta(days=2) - with pytest.raises(FilterError): - initiate_pepdb_con.annotation.get( - namespace=namespace, - query=query, - admin="private_test", - filter_by="incorrect", - filter_start_date=date_old.strftime("%Y/%m/%d"), - filter_end_date=date_now.strftime("%Y/%m/%d"), - ) - - @pytest.mark.parametrize( - "rp_list, admin, found_number", - [ - [ - [ - "namespace1/amendments1:default", - "namespace1/amendments2:default", - "namespace2/derive:default", - "private_test/amendments1:default", - ], - "namespace1", - 4, - ], - [ - [ - "namespace1/amendments1:default", - "namespace1/amendments2:default", - "namespace2/derive:default", - "private_test/amendments1:default", - ], - "private_test", - 4, - ], - ], - ) - def test_get_annotation_by_rp_list(self, initiate_pepdb_con, rp_list, admin, found_number): - result = initiate_pepdb_con.annotation.get_by_rp_list(rp_list) - assert len(result.results) == found_number - - def test_get_annotation_by_rp_enpty_list(self, initiate_pepdb_con): - result = initiate_pepdb_con.annotation.get_by_rp_list([]) - assert len(result.results) == 0 - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - ], - ) - def test_search_incorrect_incorrect_pep_type( - self, initiate_pepdb_con, namespace, query, found_number - ): - with pytest.raises(ValueError): - initiate_pepdb_con.annotation.get(namespace=namespace, pep_type="incorrect") - - @pytest.mark.parametrize( - "namespace, query, found_number", - [ - ["namespace1", "ame", 2], - ], - ) - def test_project_list_without_annotation( - self, initiate_pepdb_con, namespace, query, found_number - ): - result = initiate_pepdb_con.annotation.get_projects_list( - namespace=namespace, - search_str=query, - ) - assert len(result) == found_number - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestNamespace: - """ - Test function within namespace class - """ - - def test_annotation(self, initiate_pepdb_con): - result = initiate_pepdb_con.namespace.get() - assert len(result.results) == 3 - - def test_annotation_private(self, initiate_pepdb_con): - result = initiate_pepdb_con.namespace.get(admin="private_test") - assert len(result.results) == 4 - - def test_namespace_info(self, initiate_pepdb_con): - initiate_pepdb_con.project.update( - namespace="private_test", - name="derive", - tag="default", - update_dict={"is_private": False}, - ) - result = initiate_pepdb_con.namespace.info() - assert len(result.results) == 4 - assert result.results[3].number_of_projects == 1 - - def test_namespace_stats(self, initiate_pepdb_con): - stat_result = initiate_pepdb_con.namespace.stats(monthly=True) - assert next(iter(stat_result.projects_created.values()), 0) == 30 - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestFavorites: - """ - Test function within user class - """ - - def test_add_projects_to_favorites(self, initiate_pepdb_con): - result = initiate_pepdb_con.annotation.get( - namespace="namespace1", - ) - for project in result.results: - initiate_pepdb_con.user.add_project_to_favorites( - "random_namespace", project.namespace, project.name, "default" - ) - fav_results = initiate_pepdb_con.user.get_favorites("random_namespace") - - assert fav_results.count == len(result.results) - - # This can fail if the order of the results is different - assert fav_results.results[0].namespace == result.results[0].namespace - - def test_count_project_none(self, initiate_pepdb_con): - result = initiate_pepdb_con.user.get_favorites("private_test") - assert result.count == 0 - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_count_project_one(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.user.add_project_to_favorites(namespace, namespace, name, "default") - result = initiate_pepdb_con.user.get_favorites("namespace1") - assert result.count == 1 - result1 = initiate_pepdb_con.user.get_favorites("private_test") - assert result1.count == 0 - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_remove_from_favorite(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.user.add_project_to_favorites("namespace1", namespace, name, "default") - initiate_pepdb_con.user.add_project_to_favorites( - "namespace1", namespace, "amendments2", "default" - ) - result = initiate_pepdb_con.user.get_favorites("namespace1") - assert result.count == len(result.results) == 2 - initiate_pepdb_con.user.remove_project_from_favorites( - "namespace1", namespace, name, "default" - ) - result = initiate_pepdb_con.user.get_favorites("namespace1") - assert result.count == len(result.results) == 1 - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_remove_from_favorite_error(self, initiate_pepdb_con, namespace, name): - with pytest.raises(ProjectNotInFavorites): - initiate_pepdb_con.user.remove_project_from_favorites( - "namespace1", namespace, name, "default" - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_favorites_duplication_error(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.user.add_project_to_favorites("namespace1", namespace, name, "default") - with pytest.raises(ProjectAlreadyInFavorites): - initiate_pepdb_con.user.add_project_to_favorites( - "namespace1", namespace, name, "default" - ) - - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "amendments1"], - ], - ) - def test_annotation_favorite_number(self, initiate_pepdb_con, namespace, name): - initiate_pepdb_con.user.add_project_to_favorites("namespace1", namespace, name, "default") - annotations_in_namespace = initiate_pepdb_con.annotation.get("namespace1") - - for prj_annot in annotations_in_namespace.results: - if prj_annot.name == name: - assert prj_annot.stars_number == 1 - else: - assert prj_annot.stars_number == 0 - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestSamples: - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_retrieve_one_sample(self, initiate_pepdb_con, namespace, name, sample_name): - one_sample = initiate_pepdb_con.sample.get(namespace, name, sample_name) - assert isinstance(one_sample, peppy.Sample) - assert one_sample.sample_name == sample_name - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_retrieve_raw_sample(self, initiate_pepdb_con, namespace, name, sample_name): - one_sample = initiate_pepdb_con.sample.get(namespace, name, sample_name, raw=True) - assert isinstance(one_sample, dict) - assert one_sample["sample_name"] == sample_name - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace2", "custom_index", "frog_1"], - ], - ) - def test_retrieve_sample_with_modified_sample_id( - self, initiate_pepdb_con, namespace, name, sample_name - ): - one_sample = initiate_pepdb_con.sample.get(namespace, name, sample_name) - assert isinstance(one_sample, peppy.Sample) - assert one_sample.sample_id == "frog_1" - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_update(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.sample.update( - namespace=namespace, - name=name, - tag="default", - sample_name=sample_name, - update_dict={"organism": "butterfly"}, - ) - one_sample = initiate_pepdb_con.sample.get(namespace, name, sample_name) - assert one_sample.organism == "butterfly" - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_update_sample_name(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.sample.update( - namespace=namespace, - name=name, - tag="default", - sample_name=sample_name, - update_dict={"sample_name": "butterfly"}, - ) - one_sample = initiate_pepdb_con.sample.get(namespace, name, "butterfly") - assert one_sample.sample_name == "butterfly" - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace2", "custom_index", "frog_1"], - ], - ) - def test_update_custom_sample_id(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.sample.update( - namespace=namespace, - name=name, - tag="default", - sample_name=sample_name, - update_dict={"sample_id": "butterfly"}, - ) - one_sample = initiate_pepdb_con.sample.get(namespace, name, "butterfly") - assert one_sample.sample_id == "butterfly" - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_add_new_attributes(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.sample.update( - namespace=namespace, - name=name, - tag="default", - sample_name=sample_name, - update_dict={"new_attr": "butterfly"}, - ) - prj = initiate_pepdb_con.project.get(namespace, name) - - assert prj.get_sample(sample_name).new_attr == "butterfly" - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_project_timestamp_was_changed(self, initiate_pepdb_con, namespace, name, sample_name): - annotation1 = initiate_pepdb_con.annotation.get(namespace, name, "default") - import time - - time.sleep(0.2) - initiate_pepdb_con.sample.update( - namespace=namespace, - name=name, - tag="default", - sample_name=sample_name, - update_dict={"new_attr": "butterfly"}, - ) - annotation2 = initiate_pepdb_con.annotation.get(namespace, name, "default") - - assert annotation1.results[0].last_update_date != annotation2.results[0].last_update_date - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_delete_sample(self, initiate_pepdb_con, namespace, name, sample_name): - one_sample = initiate_pepdb_con.sample.get(namespace, name, sample_name) - assert isinstance(one_sample, peppy.Sample) - - initiate_pepdb_con.sample.delete(namespace, name, tag="default", sample_name=sample_name) - - with pytest.raises(SampleNotFoundError): - initiate_pepdb_con.sample.get(namespace, name, tag="default", sample_name=sample_name) - - @pytest.mark.parametrize( - "namespace, name, tag, sample_dict", - [ - [ - "namespace1", - "amendments1", - "default", - { - "sample_name": "new_sample", - "time": "new_time", - }, - ], - ], - ) - def test_add_sample(self, initiate_pepdb_con, namespace, name, tag, sample_dict): - prj = initiate_pepdb_con.project.get(namespace, name) - initiate_pepdb_con.sample.add(namespace, name, tag, sample_dict) - - prj2 = initiate_pepdb_con.project.get(namespace, name) - - assert len(prj.samples) + 1 == len(prj2.samples) - assert prj2.samples[-1].sample_name == sample_dict["sample_name"] - - @pytest.mark.parametrize( - "namespace, name, tag, sample_dict", - [ - [ - "namespace1", - "amendments1", - "default", - { - "sample_name": "pig_0h", - "time": "new_time", - }, - ], - ], - ) - def test_overwrite_sample(self, initiate_pepdb_con, namespace, name, tag, sample_dict): - assert initiate_pepdb_con.project.get(namespace, name).get_sample("pig_0h").time == "0" - initiate_pepdb_con.sample.add(namespace, name, tag, sample_dict, overwrite=True) - - assert ( - initiate_pepdb_con.project.get(namespace, name).get_sample("pig_0h").time == "new_time" - ) - - @pytest.mark.parametrize( - "namespace, name, tag, sample_dict", - [ - [ - "namespace1", - "amendments1", - "default", - { - "sample_name": "new_sample", - "time": "new_time", - }, - ], - ], - ) - def test_delete_and_add(self, initiate_pepdb_con, namespace, name, tag, sample_dict): - prj = initiate_pepdb_con.project.get(namespace, name) - sample_dict = initiate_pepdb_con.sample.get(namespace, name, "pig_0h", raw=True) - initiate_pepdb_con.sample.delete(namespace, name, tag, "pig_0h") - initiate_pepdb_con.sample.add(namespace, name, tag, sample_dict) - prj2 = initiate_pepdb_con.project.get(namespace, name) - assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() - - -@pytest.mark.skipif( - not db_setup(), - reason="DB is not setup", -) -class TestViews: - """ - Test function within view class - """ - - @pytest.mark.parametrize( - "namespace, name, sample_name, view_name", - [ - ["namespace1", "amendments1", "pig_0h", "view1"], - ], - ) - def test_create_view(self, initiate_pepdb_con, namespace, name, sample_name, view_name): - initiate_pepdb_con.view.create( - view_name, - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name, "pig_1h"], - }, - ) - - project = initiate_pepdb_con.project.get(namespace, name) - view_project = initiate_pepdb_con.view.get(namespace, name, "default", view_name) - assert len(view_project.samples) == 2 - assert view_project != project - - @pytest.mark.parametrize( - "namespace, name, sample_name, view_name", - [ - ["namespace1", "amendments1", "pig_0h", "view1"], - ], - ) - def test_create_view_with_incorrect_sample( - self, initiate_pepdb_con, namespace, name, sample_name, view_name - ): - with pytest.raises(SampleNotFoundError): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": "namespace1", - "project_name": "amendments1", - "project_tag": "default", - "sample_list": ["pig_0h", "pig_1h", "pig_2h"], - }, - ) - - @pytest.mark.parametrize( - "namespace, name, sample_name, view_name", - [ - ["namespace1", "amendments1", "pig_0h", "view1"], - ], - ) - def test_create_view_with_incorrect_sample_no_fail( - self, initiate_pepdb_con, namespace, name, sample_name, view_name - ): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": "namespace1", - "project_name": "amendments1", - "project_tag": "default", - "sample_list": ["pig_0h", "pig_1h", "pig_2h"], - }, - no_fail=True, - ) - project = initiate_pepdb_con.project.get(namespace, name) - view_project = initiate_pepdb_con.view.get(namespace, name, "default", view_name) - assert len(view_project.samples) == 2 - assert view_project != project - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_delete_view(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name, "pig_1h"], - }, - ) - assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 2 - initiate_pepdb_con.view.delete(namespace, name, "default", "view1") - with pytest.raises(ViewNotFoundError): - initiate_pepdb_con.view.get(namespace, name, "default", "view1") - assert len(initiate_pepdb_con.project.get(namespace, name).samples) == 4 - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_add_sample_to_view(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name], - }, - ) - initiate_pepdb_con.view.add_sample(namespace, name, "default", "view1", "pig_1h") - assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 2 - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_add_multiple_samples_to_view(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name], - }, - ) - initiate_pepdb_con.view.add_sample( - namespace, name, "default", "view1", ["pig_1h", "frog_0h"] - ) - assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 3 - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_remove_sample_from_view(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name, "pig_1h"], - }, - ) - initiate_pepdb_con.view.remove_sample(namespace, name, "default", "view1", sample_name) - assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 1 - assert len(initiate_pepdb_con.project.get(namespace, name).samples) == 4 - - with pytest.raises(SampleNotInViewError): - initiate_pepdb_con.view.remove_sample(namespace, name, "default", "view1", sample_name) - - @pytest.mark.parametrize( - "namespace, name, sample_name", - [ - ["namespace1", "amendments1", "pig_0h"], - ], - ) - def test_add_existing_sample_in_view(self, initiate_pepdb_con, namespace, name, sample_name): - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name, "pig_1h"], - }, - ) - with pytest.raises(SampleAlreadyInView): - initiate_pepdb_con.view.add_sample(namespace, name, "default", "view1", sample_name) - - @pytest.mark.parametrize( - "namespace, name, sample_name, view_name", - [ - ["namespace1", "amendments1", "pig_0h", "view1"], - ], - ) - def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name, view_name): - snap_project = initiate_pepdb_con.view.get_snap_view( - namespace=namespace, - name=name, - tag="default", - sample_name_list=[sample_name, "pig_1h"], - ) - - assert len(snap_project.samples) == 2 - - @pytest.mark.parametrize( - "namespace, name, sample_name, view_name", - [ - ["namespace1", "amendments1", "pig_0h", "view1"], - ], - ) - def test_get_view_list_from_project( - self, initiate_pepdb_con, namespace, name, sample_name, view_name - ): - assert ( - len(initiate_pepdb_con.view.get_views_annotation(namespace, name, "default").views) - == 0 - ) - initiate_pepdb_con.view.create( - "view1", - { - "project_namespace": namespace, - "project_name": name, - "project_tag": "default", - "sample_list": [sample_name, "pig_1h"], - }, - ) - assert ( - len(initiate_pepdb_con.view.get_views_annotation(namespace, name, "default").views) - == 1 - ) diff --git a/tests/test_project.py b/tests/test_project.py new file mode 100644 index 0000000..26cf017 --- /dev/null +++ b/tests/test_project.py @@ -0,0 +1,301 @@ +import numpy as np +import peppy +import pytest + +from pepdbagent.exceptions import ProjectNotFoundError + +from .utils import PEPDBAgentContextManager, get_path_to_example_file, list_of_available_peps + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestProject: + """ + Test project methods + """ + + def test_create_project(self): + with PEPDBAgentContextManager(add_data=False) as agent: + prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + agent.project.create(prj, namespace="test", name="imply", overwrite=True) + assert True + + def test_create_project_from_dict(self): + with PEPDBAgentContextManager(add_data=False) as agent: + prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + agent.project.create( + prj.to_dict(extended=True, orient="records"), + namespace="test", + name="imply", + overwrite=True, + ) + assert True + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ["namespace1", "basic"], + ["namespace2", "derive"], + ["namespace2", "imply"], + ["namespace3", "piface"], + ["namespace3", "subtable2"], + ], + ) + def test_get_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + kk = agent.project.get(namespace=namespace, name=name, tag="default", raw=False) + ff = peppy.Project(get_path_to_example_file(namespace, name)) + assert kk == ff + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_get_config(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + description = "" + kk = agent.project.get_config( + namespace=namespace, + name=name, + tag="default", + ) + ff = peppy.Project(get_path_to_example_file(namespace, name)) + ff.description = description + ff.name = name + assert kk == ff.config + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace3", "subtables"], + ], + ) + def test_get_subsamples(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj_subtables = agent.project.get_subsamples( + namespace=namespace, + name=name, + tag="default", + ) + orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + + assert ( + prj_subtables + == orgiginal_prj.to_dict(extended=True, orient="records")["_subsample_list"] + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace3", "subtables"], + ], + ) + def test_get_samples_raw(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj_samples = agent.project.get_samples( + namespace=namespace, name=name, tag="default", raw=True + ) + orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + + assert ( + prj_samples + == orgiginal_prj.to_dict(extended=True, orient="records")["_sample_dict"] + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace3", "subtables"], + ], + ) + def test_get_samples_processed(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj_samples = agent.project.get_samples( + namespace=namespace, + name=name, + tag="default", + raw=False, + ) + orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + + assert prj_samples == orgiginal_prj.sample_table.replace({np.nan: None}).to_dict( + orient="records" + ) + + @pytest.mark.parametrize( + "namespace, name,tag", + [ + ["incorrect_namespace", "amendments1", "default"], + ["namespace1", "subtable2", "default"], + ["namespace3", "basic", "default"], + ["namespace3", "subtable2", "incorrect_tag"], + ["namespace1", "incorrect_name", "default"], + ], + ) + def test_get_project_error(self, namespace, name, tag): + with PEPDBAgentContextManager(add_data=True) as agent: + with pytest.raises(ProjectNotFoundError, match="Project does not exist."): + agent.project.get(namespace=namespace, name=name, tag=tag) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ["namespace2", "derive"], + ["namespace2", "imply"], + ], + ) + def test_overwrite_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + new_prj = agent.project.get(namespace="namespace1", name="basic", raw=False) + + agent.project.create( + project=new_prj, + namespace=namespace, + name=name, + tag="default", + overwrite=True, + ) + + assert agent.project.get(namespace=namespace, name=name, raw=False) == new_prj + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace2", "derive"], + ], + ) + def test_delete_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.delete(namespace=namespace, name=name, tag="default") + + with pytest.raises(ProjectNotFoundError, match="Project does not exist."): + agent.project.get(namespace=namespace, name=name, tag="default") + + def test_delete_not_existing_project(self): + with PEPDBAgentContextManager(add_data=True) as agent: + with pytest.raises(ProjectNotFoundError, match="Project does not exist."): + agent.project.delete(namespace="namespace1", name="nothing", tag="default") + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments2"], + ["namespace2", "derive"], + ["namespace2", "imply"], + ], + ) + def test_fork_projects(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.fork( + original_namespace=namespace, + original_name=name, + original_tag="default", + fork_namespace="new_namespace", + fork_name="new_name", + fork_tag="new_tag", + ) + + assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + result = agent.annotation.get( + namespace="new_namespace", name="new_name", tag="new_tag" + ) + assert result.results[0].forked_from == f"{namespace}/{name}:default" + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ], + ) + def test_parent_project_delete(self, namespace, name): + """ + Test if parent project is deleted, forked project is not deleted + """ + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.fork( + original_namespace=namespace, + original_name=name, + original_tag="default", + fork_namespace="new_namespace", + fork_name="new_name", + fork_tag="new_tag", + ) + + assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + agent.project.delete(namespace=namespace, name=name, tag="default") + assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ], + ) + def test_child_project_delete(self, namespace, name): + """ + Test if child project is deleted, parent project is not deleted + """ + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.fork( + original_namespace=namespace, + original_name=name, + original_tag="default", + fork_namespace="new_namespace", + fork_name="new_name", + fork_tag="new_tag", + ) + + assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists(namespace=namespace, name=name, tag="default") + agent.project.delete(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists(namespace=namespace, name=name, tag="default") + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace1", "amendments2"], + ], + ) + def test_project_can_be_forked_twice(self, namespace, name): + """ + Test if project can be forked twice + """ + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.fork( + original_namespace=namespace, + original_name=name, + original_tag="default", + fork_namespace="new_namespace", + fork_name="new_name", + fork_tag="new_tag", + ) + agent.project.fork( + original_namespace=namespace, + original_name=name, + original_tag="default", + fork_namespace="new_namespace2", + fork_name="new_name2", + fork_tag="new_tag2", + ) + + result = agent.annotation.get( + namespace="new_namespace", name="new_name", tag="new_tag" + ) + assert result.results[0].forked_from == f"{namespace}/{name}:default" + + result = agent.annotation.get( + namespace="new_namespace2", name="new_name2", tag="new_tag2" + ) + assert result.results[0].forked_from == f"{namespace}/{name}:default" diff --git a/tests/test_samples.py b/tests/test_samples.py new file mode 100644 index 0000000..0e4de49 --- /dev/null +++ b/tests/test_samples.py @@ -0,0 +1,234 @@ +import peppy +import pytest +from pepdbagent.exceptions import SampleNotFoundError + +from .utils import PEPDBAgentContextManager + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestSamples: + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_retrieve_one_sample(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + one_sample = agent.sample.get(namespace, name, sample_name, raw=False) + assert isinstance(one_sample, peppy.Sample) + assert one_sample.sample_name == sample_name + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_retrieve_raw_sample(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + one_sample = agent.sample.get(namespace, name, sample_name) + assert isinstance(one_sample, dict) + assert one_sample["sample_name"] == sample_name + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace2", "custom_index", "frog_1"], + ], + ) + def test_retrieve_sample_with_modified_sample_id(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + one_sample = agent.sample.get(namespace, name, sample_name, raw=False) + assert isinstance(one_sample, peppy.Sample) + assert one_sample.sample_id == "frog_1" + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_update(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.sample.update( + namespace=namespace, + name=name, + tag="default", + sample_name=sample_name, + update_dict={"organism": "butterfly"}, + ) + one_sample = agent.sample.get(namespace, name, sample_name, raw=False) + assert one_sample.organism == "butterfly" + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_update_sample_name(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.sample.update( + namespace=namespace, + name=name, + tag="default", + sample_name=sample_name, + update_dict={"sample_name": "butterfly"}, + ) + one_sample = agent.sample.get(namespace, name, "butterfly", raw=False) + assert one_sample.sample_name == "butterfly" + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace2", "custom_index", "frog_1"], + ], + ) + def test_update_custom_sample_id(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.sample.update( + namespace=namespace, + name=name, + tag="default", + sample_name=sample_name, + update_dict={"sample_id": "butterfly"}, + ) + one_sample = agent.sample.get(namespace, name, "butterfly", raw=False) + assert one_sample.sample_id == "butterfly" + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_new_attributes(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.sample.update( + namespace=namespace, + name=name, + tag="default", + sample_name=sample_name, + update_dict={"new_attr": "butterfly"}, + ) + prj = agent.project.get(namespace, name, raw=False) + + assert prj.get_sample(sample_name).new_attr == "butterfly" + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_project_timestamp_was_changed(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + annotation1 = agent.annotation.get(namespace, name, "default") + import time + + time.sleep(0.2) + agent.sample.update( + namespace=namespace, + name=name, + tag="default", + sample_name=sample_name, + update_dict={"new_attr": "butterfly"}, + ) + annotation2 = agent.annotation.get(namespace, name, "default") + + assert ( + annotation1.results[0].last_update_date != annotation2.results[0].last_update_date + ) + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_delete_sample(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + one_sample = agent.sample.get(namespace, name, sample_name, raw=False) + assert isinstance(one_sample, peppy.Sample) + + agent.sample.delete(namespace, name, tag="default", sample_name=sample_name) + + with pytest.raises(SampleNotFoundError): + agent.sample.get( + namespace, name, tag="default", sample_name=sample_name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name, tag, sample_dict", + [ + [ + "namespace1", + "amendments1", + "default", + { + "sample_name": "new_sample", + "time": "new_time", + }, + ], + ], + ) + def test_add_sample(self, namespace, name, tag, sample_dict): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, raw=False) + agent.sample.add(namespace, name, tag, sample_dict) + + prj2 = agent.project.get(namespace, name, raw=False) + + assert len(prj.samples) + 1 == len(prj2.samples) + assert prj2.samples[-1].sample_name == sample_dict["sample_name"] + + @pytest.mark.parametrize( + "namespace, name, tag, sample_dict", + [ + [ + "namespace1", + "amendments1", + "default", + { + "sample_name": "pig_0h", + "time": "new_time", + }, + ], + ], + ) + def test_overwrite_sample(self, namespace, name, tag, sample_dict): + with PEPDBAgentContextManager(add_data=True) as agent: + assert agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time == "0" + agent.sample.add(namespace, name, tag, sample_dict, overwrite=True) + + assert ( + agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time + == "new_time" + ) + + @pytest.mark.parametrize( + "namespace, name, tag, sample_dict", + [ + [ + "namespace1", + "amendments1", + "default", + { + "sample_name": "new_sample", + "time": "new_time", + }, + ], + ], + ) + def test_delete_and_add(self, namespace, name, tag, sample_dict): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace, name, raw=False) + sample_dict = agent.sample.get(namespace, name, "pig_0h", raw=True) + agent.sample.delete(namespace, name, tag, "pig_0h") + agent.sample.add(namespace, name, tag, sample_dict) + prj2 = agent.project.get(namespace, name, raw=False) + assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() diff --git a/tests/test_updates.py b/tests/test_updates.py new file mode 100644 index 0000000..73854c9 --- /dev/null +++ b/tests/test_updates.py @@ -0,0 +1,606 @@ +import peppy +import pytest +from peppy.exceptions import IllegalStateException +from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError +from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY + +from .utils import PEPDBAgentContextManager + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestProjectUpdate: + @pytest.mark.parametrize( + "namespace, name,new_name", + [ + ["namespace1", "amendments1", "name1"], + ["namespace1", "amendments2", "name2"], + ], + ) + def test_update_project_name(self, namespace, name, new_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"name": new_name}, + ) + assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + + @pytest.mark.parametrize( + "namespace, name,new_name", + [ + ["namespace1", "amendments1", "name1"], + ["namespace1", "amendments2", "name2"], + ], + ) + def test_update_project_name_in_config(self, namespace, name, new_name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj.name = new_name + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": prj}, + ) + assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + + @pytest.mark.parametrize( + "namespace, name, new_tag", + [ + ["namespace1", "amendments1", "tag1"], + ["namespace1", "amendments2", "tag2"], + ], + ) + def test_update_project_tag(self, namespace, name, new_tag): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"tag": new_tag}, + ) + assert agent.project.exists(namespace=namespace, name=name, tag=new_tag) + + @pytest.mark.parametrize( + "namespace, name, new_description", + [ + ["namespace1", "amendments1", "desc1 f"], + ["namespace2", "derive", "desc5 f"], + ], + ) + def test_update_project_description(self, namespace, name, new_description): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace=namespace, name=name, raw=False) + prj.description = new_description + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"description": new_description}, + ) + + assert ( + agent.project.get(namespace=namespace, name=name, raw=False).description + == new_description + ) + + @pytest.mark.parametrize( + "namespace, name, new_description", + [ + ["namespace1", "amendments1", "desc1 f"], + ], + ) + def test_update_project_description_in_config(self, namespace, name, new_description): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj.description = new_description + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": prj}, + ) + + assert ( + agent.project.get(namespace=namespace, name=name, raw=False).description + == new_description + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_update_whole_project(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + new_prj = agent.project.get(namespace="namespace1", name="basic", raw=False) + # update name. If name is different, it will update name too + new_prj.name = name + with pytest.raises(SampleTableUpdateError): + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": new_prj}, + ) + + @pytest.mark.parametrize( + "namespace, name, pep_schema", + [ + ["namespace1", "amendments1", "schema1"], + ["namespace2", "derive", "schema3"], + ], + ) + def test_update_pep_schema(self, namespace, name, pep_schema): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"pep_schema": pep_schema}, + ) + res = agent.annotation.get(namespace, name, "default") + assert res.results[0].pep_schema == pep_schema + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_update_project_private(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"is_private": True}, + ) + + is_private = ( + agent.annotation.get( + namespace=namespace, name=name, tag="default", admin=[namespace] + ) + .results[0] + .is_private + ) + assert is_private is True + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_update_project_pop(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"pop": True}, + ) + + pop = ( + agent.annotation.get( + namespace=namespace, name=name, tag="default", admin=[namespace] + ) + .results[0] + .pop + ) + assert pop is True + + # Update to pop = False and check if it is updated + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"pop": False}, + ) + + pop = ( + agent.annotation.get( + namespace=namespace, name=name, tag="default", admin=[namespace] + ) + .results[0] + .pop + ) + assert pop is False + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "basic"], + ], + ) + def test_project_can_have_2_sample_names(self, namespace, name): + """ + In PEP 2.1.0 project can have 2 rows with same sample name, + ensure that update works correctly + """ + with PEPDBAgentContextManager(add_data=True) as agent: + new_prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj_dict = new_prj.to_dict(extended=True, orient="records") + + prj_dict["_sample_dict"].append( + { + "file": "data/frog23_data.txt", + "protocol": "anySample3Type", + "sample_name": "frog_2", + } + ) + prj_dict["_sample_dict"].append( + { + "file": "data/frog23_data.txt4", + "protocol": "anySample3Type4", + "sample_name": "frog_2", + } + ) + + new_prj.description = "new_description" + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj_dict)}, + ) + + prj = agent.project.get(namespace=namespace, name=name, raw=True) + + assert len(prj["_sample_dict"]) == 4 + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestUpdateProjectWithId: + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_update_whole_project_with_id(self, namespace, name): + """ + General test for updating whole project with id (inserting one project without id) + """ + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + new_sample = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample.copy()) + prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + del prj["_sample_dict"][1] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + del new_sample[PEPHUB_SAMPLE_ID_KEY] + peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id + peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample + del peppy_prj["_sample_dict"][1] # delete sample + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_insert_new_row(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + new_sample = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + del new_sample[PEPHUB_SAMPLE_ID_KEY] + peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_insert_new_multiple_rows(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + new_sample2 = { + "sample_name": "new_sample2", + "protocol": "new_protocol2", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample1.copy()) + prj["_sample_dict"].append(new_sample2.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + del new_sample1[PEPHUB_SAMPLE_ID_KEY] + del new_sample2[PEPHUB_SAMPLE_ID_KEY] + peppy_prj["_sample_dict"].append(new_sample1.copy()) # add sample without id + peppy_prj["_sample_dict"].append(new_sample2.copy()) # add sample without id + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace3", "subtable1"], + ], + ) + def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + new_sample1 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + new_sample2 = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].append(new_sample1.copy()) + prj["_sample_dict"].append(new_sample2.copy()) + + with pytest.raises(IllegalStateException): + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace3", "subtable2"], + ["namespace1", "append"], + ], + ) + def test_delete_multiple_rows(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + del prj["_sample_dict"][1] + del prj["_sample_dict"][2] + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + del peppy_prj["_sample_dict"][1] # delete sample + del peppy_prj["_sample_dict"][2] # delete sample + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_modify_one_row(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_modify_multiple_rows(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample + peppy_prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" # modify sample + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_add_new_first_sample(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + new_sample = { + "sample_name": "new_sample", + "protocol": "new_protocol", + PEPHUB_SAMPLE_ID_KEY: None, + } + + prj["_sample_dict"].insert(0, new_sample.copy()) + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + del new_sample[PEPHUB_SAMPLE_ID_KEY] + peppy_prj["_sample_dict"].insert(0, new_sample.copy()) # add sample without id + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_change_sample_order(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + + sample1 = prj["_sample_dict"][0].copy() + sample2 = prj["_sample_dict"][1].copy() + + prj["_sample_dict"][0] = sample2 + prj["_sample_dict"][1] = sample1 + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + peppy_prj["_sample_dict"][0] = sample2 + peppy_prj["_sample_dict"][1] = sample1 + + del peppy_prj["_sample_dict"][0][PEPHUB_SAMPLE_ID_KEY] + del peppy_prj["_sample_dict"][1][PEPHUB_SAMPLE_ID_KEY] + + assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + namespace=namespace, name=name, raw=False + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ["namespace3", "subtable1"], + ], + ) + def test_update_porject_without_ids(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=False) + + prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + + with pytest.raises(SampleTableUpdateError): + + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(prj)}, + ) + + @pytest.mark.parametrize( + "namespace, name", + [ + ["namespace1", "amendments1"], + ], + ) + def test_update_project_with_duplicated_sample_guids(self, namespace, name): + with PEPDBAgentContextManager(add_data=True) as agent: + new_prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + new_prj["_sample_dict"].append(new_prj["_sample_dict"][0]) + + with pytest.raises(ProjectDuplicatedSampleGUIDsError): + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peppy.Project.from_dict(new_prj)}, + ) diff --git a/tests/test_views.py b/tests/test_views.py new file mode 100644 index 0000000..475936b --- /dev/null +++ b/tests/test_views.py @@ -0,0 +1,231 @@ +import pytest + +from pepdbagent.exceptions import ( + SampleAlreadyInView, + SampleNotFoundError, + SampleNotInViewError, + ViewNotFoundError, +) + +from .utils import PEPDBAgentContextManager + + +@pytest.mark.skipif( + not PEPDBAgentContextManager().db_setup(), + reason="DB is not setup", +) +class TestViews: + """ + Test function within view class + """ + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_create_view(self, namespace, name, sample_name, view_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + view_name, + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + + project = agent.project.get(namespace, name, raw=False) + view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + assert len(view_project.samples) == 2 + assert view_project != project + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_create_view_with_incorrect_sample(self, namespace, name, sample_name, view_name): + with PEPDBAgentContextManager(add_data=True) as agent: + with pytest.raises(SampleNotFoundError): + agent.view.create( + "view1", + { + "project_namespace": "namespace1", + "project_name": "amendments1", + "project_tag": "default", + "sample_list": ["pig_0h", "pig_1h", "pig_2h"], + }, + ) + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_create_view_with_incorrect_sample_no_fail( + self, namespace, name, sample_name, view_name + ): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": "namespace1", + "project_name": "amendments1", + "project_tag": "default", + "sample_list": ["pig_0h", "pig_1h", "pig_2h"], + }, + no_fail=True, + ) + project = agent.project.get(namespace, name, raw=False) + view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + assert len(view_project.samples) == 2 + assert view_project != project + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_delete_view(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + agent.view.delete(namespace, name, "default", "view1") + with pytest.raises(ViewNotFoundError): + agent.view.get(namespace, name, "default", "view1", raw=False) + assert len(agent.project.get(namespace, name, raw=False).samples) == 4 + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_sample_to_view(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name], + }, + ) + agent.view.add_sample(namespace, name, "default", "view1", "pig_1h") + assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_multiple_samples_to_view(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name], + }, + ) + agent.view.add_sample(namespace, name, "default", "view1", ["pig_1h", "frog_0h"]) + assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 3 + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_remove_sample_from_view(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + agent.view.remove_sample(namespace, name, "default", "view1", sample_name) + assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 1 + assert len(agent.project.get(namespace, name, raw=False).samples) == 4 + + with pytest.raises(SampleNotInViewError): + agent.view.remove_sample(namespace, name, "default", "view1", sample_name) + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_existing_sample_in_view(self, namespace, name, sample_name): + with PEPDBAgentContextManager(add_data=True) as agent: + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + with pytest.raises(SampleAlreadyInView): + agent.view.add_sample(namespace, name, "default", "view1", sample_name) + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_get_snap_view(self, namespace, name, sample_name, view_name): + with PEPDBAgentContextManager(add_data=True) as agent: + snap_project = agent.view.get_snap_view( + namespace=namespace, + name=name, + tag="default", + sample_name_list=[sample_name, "pig_1h"], + ) + + assert len(snap_project.samples) == 2 + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_get_view_list_from_project(self, namespace, name, sample_name, view_name): + with PEPDBAgentContextManager(add_data=True) as agent: + assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 0 + agent.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 1 diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..6e3e7f4 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,93 @@ +import os +import peppy +import warnings +from sqlalchemy.exc import OperationalError + +from pepdbagent import PEPDatabaseAgent + +DSN = "postgresql+psycopg://postgres:pass8743hf9h23f87h437@localhost:5432/pep-db" + +DATA_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "tests", + "data", +) + + +def get_path_to_example_file(namespace: str, project_name: str) -> str: + """ + Get path to example file + """ + return os.path.join(DATA_PATH, namespace, project_name, "project_config.yaml") + + +def list_of_available_peps() -> dict: + pep_namespaces = os.listdir(DATA_PATH) + projects = {} + for np in pep_namespaces: + pep_name = os.listdir(os.path.join(DATA_PATH, np)) + projects[np] = {p: get_path_to_example_file(np, p) for p in pep_name} + return projects + + +class PEPDBAgentContextManager: + """ + Class with context manager to connect to database. Adds data and drops everything from the database upon exit to ensure. + """ + + def __init__(self, url: str = DSN, add_data: bool = False): + """ + :param url: database url e.g. "postgresql+psycopg://postgres:docker@localhost:5432/pep-db" + :param add_data: add data to the database + """ + + self.url = url + self._agent = None + self.add_data = add_data + + def __enter__(self): + self._agent = PEPDatabaseAgent(dsn=self.url, echo=False) + self.db_engine = self._agent.pep_db_engine + self.db_engine.create_schema() + if self.add_data: + self._insert_data() + return self._agent + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.db_engine.delete_schema() + + def _insert_data(self): + pepdb_con = PEPDatabaseAgent(dsn=self.url, echo=True) + for namespace, item in list_of_available_peps().items(): + if namespace == "private_test": + private = True + else: + private = False + for name, path in item.items(): + prj = peppy.Project(path) + pepdb_con.project.create( + namespace=namespace, + name=name, + tag="default", + is_private=private, + project=prj, + overwrite=True, + pep_schema="random_schema_name", + ) + + @property + def agent(self) -> PEPDatabaseAgent: + return self._agent + + def db_setup(self): + # Check if the database is setup + try: + PEPDatabaseAgent(dsn=self.url) + except OperationalError: + warnings.warn( + UserWarning( + f"Skipping tests, because DB is not setup. {self.url}. To setup DB go to README.md" + ) + ) + return False + return True