Merge pull request #143 from pepkit/dev

Release 0.11.0

docs/changelog.md

@@ -2,6 +2,10 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [0.11.0] -- 2024-07-24
+- Added validation schemas
+
+
 ## [0.10.0] -- 2024-07-18
 - Added user delete method
 - Added project history and restoring projects

pepdbagent/_version.py

@@ -1 +1 @@
-__version__ = "0.10.0"
+__version__ = "0.11.0"

pepdbagent/db_utils.py

@@ -87,6 +87,12 @@ class Projects(Base):
         default=deliver_update_date,  # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites
     )
     pep_schema: Mapped[Optional[str]]
+
+    schema_id: Mapped[Optional[int]] = mapped_column(
+        ForeignKey("schemas.id", ondelete="SET NULL"), nullable=True
+    )
+    schema_mapping: Mapped["Schemas"] = relationship("Schemas", lazy="joined")
+
     pop: Mapped[Optional[bool]] = mapped_column(default=False)
     samples_mapping: Mapped[List["Samples"]] = relationship(
         back_populates="project_mapping", cascade="all, delete-orphan"
@@ -123,7 +129,7 @@ class Projects(Base):
 
     history_mapping: Mapped[List["HistoryProjects"]] = relationship(
         back_populates="project_mapping", cascade="all, delete-orphan"
-    )  # TODO: check if cascade is correct
+    )
 
     __table_args__ = (UniqueConstraint("namespace", "name", "tag"),)
 
@@ -296,6 +302,68 @@ class HistorySamples(Base):
     )
 
 
+class Schemas(Base):
+
+    __tablename__ = "schemas"
+
+    id: Mapped[int] = mapped_column(primary_key=True, index=True)
+    namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE"))
+    name: Mapped[str] = mapped_column(nullable=False, index=True)
+    description: Mapped[Optional[str]] = mapped_column(nullable=True, index=True)
+    schema_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue())
+    private: Mapped[bool] = mapped_column(default=False)
+    submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date)
+    last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column(
+        default=deliver_update_date, onupdate=deliver_update_date
+    )
+
+    projects_mappings: Mapped[List["Projects"]] = relationship(
+        "Projects", back_populates="schema_mapping"
+    )
+    group_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship(
+        "SchemaGroupRelations", back_populates="schema_mapping"
+    )
+
+    __table_args__ = (UniqueConstraint("namespace", "name"),)
+
+
+class SchemaGroups(Base):
+
+    __tablename__ = "schema_groups"
+
+    id: Mapped[int] = mapped_column(primary_key=True, index=True)
+    namespace: Mapped[str] = mapped_column(
+        ForeignKey("users.namespace", ondelete="CASCADE"), index=True
+    )
+    name: Mapped[str] = mapped_column(nullable=False, index=True)
+    description: Mapped[Optional[str]] = mapped_column(nullable=True)
+
+    schema_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship(
+        "SchemaGroupRelations", back_populates="group_mapping"
+    )
+
+    __table_args__ = (UniqueConstraint("namespace", "name"),)
+
+
+class SchemaGroupRelations(Base):
+
+    __tablename__ = "schema_group_relations"
+
+    schema_id: Mapped[int] = mapped_column(
+        ForeignKey("schemas.id", ondelete="CASCADE"), index=True, primary_key=True
+    )
+    group_id: Mapped[int] = mapped_column(
+        ForeignKey("schema_groups.id", ondelete="CASCADE"), index=True, primary_key=True
+    )
+
+    schema_mapping: Mapped["Schemas"] = relationship(
+        "Schemas", back_populates="group_relation_mapping"
+    )
+    group_mapping: Mapped["SchemaGroups"] = relationship(
+        "SchemaGroups", back_populates="schema_relation_mapping"
+    )
+
+
 class BaseEngine:
     """
     A class with base methods, that are used in several classes. e.g. fetch_one or fetch_all

pepdbagent/exceptions.py

@@ -117,3 +117,33 @@ def __init__(self, msg=""):
 class UserNotFoundError(PEPDatabaseAgentError):
     def __init__(self, msg=""):
         super().__init__(f"""User does not exist. {msg}""")
+
+
+class SchemaDoesNotExistError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema does not exist. {msg}""")
+
+
+class SchemaAlreadyExistsError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema already exists. {msg}""")
+
+
+class SchemaGroupDoesNotExistError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema group does not exist. {msg}""")
+
+
+class SchemaGroupAlreadyExistsError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema group already exists. {msg}""")
+
+
+class SchemaAlreadyInGroupError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema already in the group. {msg}""")
+
+
+class SchemaIsNotInGroupError(PEPDatabaseAgentError):
+    def __init__(self, msg=""):
+        super().__init__(f"""Schema not found in group. {msg}""")

pepdbagent/models.py

@@ -99,6 +99,7 @@ class UpdateItems(BaseModel):
     samples: Optional[List[dict]] = None
     subsamples: Optional[List[List[dict]]] = None
     pop: Optional[bool] = None
+    schema_id: Optional[int] = None
 
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -246,3 +247,49 @@ class HistoryAnnotationModel(BaseModel):
     name: str
     tag: str = DEFAULT_TAG
     history: List[HistoryChangeModel]
+
+
+class SchemaAnnotation(BaseModel):
+    """
+    Schema annotation model
+    """
+
+    namespace: str
+    name: str
+    last_update_date: str
+    submission_date: str
+    description: Optional[str] = ""
+    popularity_number: Optional[int] = 0
+
+
+class SchemaSearchResult(BaseModel):
+    """
+    Schema search result model
+    """
+
+    count: int
+    limit: int
+    offset: int
+    results: List[SchemaAnnotation]
+
+
+class SchemaGroupAnnotation(BaseModel):
+    """
+    Schema group annotation model
+    """
+
+    namespace: str
+    name: str
+    description: Optional[str]
+    schemas: List[SchemaAnnotation]
+
+
+class SchemaGroupSearchResult(BaseModel):
+    """
+    Schema group search result model
+    """
+
+    count: int
+    limit: int
+    offset: int
+    results: List[SchemaGroupAnnotation]

pepdbagent/modules/annotation.py

@@ -211,7 +211,11 @@ def _get_single_annotation(
                     submission_date=str(query_result.submission_date),
                     last_update_date=str(query_result.last_update_date),
                     digest=query_result.digest,
-                    pep_schema=query_result.pep_schema,
+                    pep_schema=(
+                        f"{query_result.schema_mapping.namespace}/{query_result.schema_mapping.name}"
+                        if query_result.schema_mapping
+                        else None
+                    ),
                     pop=query_result.pop,
                     stars_number=query_result.number_of_stars,
                     forked_from=(
@@ -342,7 +346,11 @@ def _get_projects(
                         submission_date=str(result.submission_date),
                         last_update_date=str(result.last_update_date),
                         digest=result.digest,
-                        pep_schema=result.pep_schema,
+                        pep_schema=(
+                            f"{result.schema_mapping.namespace}/{result.schema_mapping.name}"
+                            if result.schema_mapping
+                            else None
+                        ),
                         pop=result.pop,
                         stars_number=result.number_of_stars,
                         forked_from=(
@@ -538,9 +546,9 @@ def get_by_rp_list(
             statement = select(Projects).where(or_(*or_statement_list))
             anno_results = []
             with Session(self._sa_engine) as session:
-                query_result = session.execute(statement).all()
+                query_result = session.scalars(statement)
                 for result in query_result:
-                    project_obj = result[0]
+                    project_obj = result
                     annot = AnnotationModel(
                         namespace=project_obj.namespace,
                         name=project_obj.name,
@@ -551,7 +559,11 @@ def get_by_rp_list(
                         submission_date=str(project_obj.submission_date),
                         last_update_date=str(project_obj.last_update_date),
                         digest=project_obj.digest,
-                        pep_schema=project_obj.pep_schema,
+                        pep_schema=(
+                            f"{project_obj.schema_mapping.namespace}/{project_obj.schema_mapping.name}"
+                            if project_obj.schema_mapping
+                            else None
+                        ),
                         pop=project_obj.pop,
                         stars_number=project_obj.number_of_stars,
                         forked_from=(

pepdbagent/modules/project.py

@@ -31,6 +31,7 @@
     HistorySamples,
     Projects,
     Samples,
+    Schemas,
     Subsamples,
     UpdateTypes,
     User,
@@ -42,6 +43,7 @@
     ProjectNotFoundError,
     ProjectUniqueNameError,
     SampleTableUpdateError,
+    SchemaDoesNotExistError,
 )
 from pepdbagent.models import (
     HistoryAnnotationModel,
@@ -50,7 +52,13 @@
     UpdateItems,
     UpdateModel,
 )
-from pepdbagent.utils import create_digest, generate_guid, order_samples, registry_path_converter
+from pepdbagent.utils import (
+    create_digest,
+    generate_guid,
+    order_samples,
+    registry_path_converter,
+    schema_path_converter,
+)
 
 _LOGGER = logging.getLogger(PKG_NAME)
 
@@ -314,7 +322,7 @@ def create(
         :param name: name of the project (Default: name is taken from the project object)
         :param tag: tag (or version) of the project.
         :param is_private: boolean value if the project should be visible just for user that creates it.
-        :param pep_schema: assign PEP to a specific schema. [Default: None]
+        :param pep_schema: assign PEP to a specific schema. Example: 'namespace/name' [Default: None]
         :param pop: if project is a pep of peps (POP) [Default: False]
         :param overwrite: if project exists overwrite the project, otherwise upload it.
             [Default: False - project won't be overwritten if it exists in db]
@@ -356,6 +364,24 @@ def create(
         except AttributeError:
             number_of_samples = len(proj_dict[SAMPLE_RAW_DICT_KEY])
 
+        if pep_schema:
+            schema_namespace, schema_name = schema_path_converter(pep_schema)
+            with Session(self._sa_engine) as session:
+                schema_mapping = session.scalar(
+                    select(Schemas).where(
+                        and_(
+                            Schemas.namespace == schema_namespace,
+                            Schemas.name == schema_name,
+                        )
+                    )
+                )
+                if not schema_mapping:
+                    raise SchemaDoesNotExistError(
+                        f"Schema {schema_namespace}/{schema_name} does not exist. "
+                        f"Project won't be uploaded."
+                    )
+                pep_schema = schema_mapping.id
+
         if update_only:
             _LOGGER.info(f"Update_only argument is set True. Updating project {proj_name} ...")
             self._overwrite(
@@ -384,7 +410,8 @@ def create(
                     private=is_private,
                     submission_date=datetime.datetime.now(datetime.timezone.utc),
                     last_update_date=datetime.datetime.now(datetime.timezone.utc),
-                    pep_schema=pep_schema,
+                    # pep_schema=pep_schema,
+                    schema_id=pep_schema,
                     description=description,
                     pop=pop,
                 )
@@ -447,7 +474,7 @@ def _overwrite(
         project_digest: str,
         number_of_samples: int,
         private: bool = False,
-        pep_schema: str = None,
+        pep_schema: int = None,
         description: str = "",
         pop: bool = False,
     ) -> None:
@@ -483,7 +510,8 @@ def _overwrite(
                     found_prj.digest = project_digest
                     found_prj.number_of_samples = number_of_samples
                     found_prj.private = private
-                    found_prj.pep_schema = pep_schema
+                    # found_prj.pep_schema = pep_schema
+                    found_prj.schema_id = pep_schema
                     found_prj.config = project_dict[CONFIG_KEY]
                     found_prj.description = description
                     found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc)
@@ -577,6 +605,8 @@ def update(
                         f"Pep {namespace}/{name}:{tag} was not found. No items will be updated!"
                     )
 
+                self._convert_update_schema_id(session, update_values)
+
                 for k, v in update_values.items():
                     if getattr(found_prj, k) != v:
                         setattr(found_prj, k, v)
@@ -647,6 +677,34 @@ def update(
         else:
             raise ProjectNotFoundError("No items will be updated!")
 
+    @staticmethod
+    def _convert_update_schema_id(session: Session, update_values: dict):
+        """
+        Convert schema path to schema_id in update_values and update it in update dict
+
+
+        :param session: open session object
+        :param update_values: dict with update key->values
+
+        return None
+        """
+        if "pep_schema" in update_values:
+            schema_namespace, schema_name = schema_path_converter(update_values["pep_schema"])
+            schema_mapping = session.scalar(
+                select(Schemas).where(
+                    and_(
+                        Schemas.namespace == schema_namespace,
+                        Schemas.name == schema_name,
+                    )
+                )
+            )
+            if not schema_mapping:
+                raise SchemaDoesNotExistError(
+                    f"Schema {schema_namespace}/{schema_name} does not exist. "
+                    f"Project won't be updated."
+                )
+            update_values["schema_id"] = schema_mapping.id
+
     def _update_samples(
         self,
         project_id: int,