Adding SDS export format as FormatterV2 (#2218)

* adding xlsx support dependencies

* base class for formatting xlsx documents

* wip: xlsx assembly and tuning

* updated docs and changed cell_stles to a list

* migrated to list from dict to allow for duplication of keys

* publishing progress

* aded annotations

* added yellow_dark and renamed colors

* code description finished

* extended to support input data ranges

* added model for submission

* added code/manifest xlsx template

* there is only one directory manifest shared between all

* refactored and split directory structure

* all sheets classes start with Sheet

* finished base dataset description

* added comment

* added default exports

* moed xlsx and added cimis structure

* restructured code
added new entrypoint for xlsx
creating a unique entrypoint for all

* added default file save names

* extended xlsx base document:
- saving is done in base_path
- also sheets are stored by name for later usage

* adding pydatic validation

* added utils and refactore

* refactored to use new format for parsing and validating

* added missing file

* added dataset_description filling methods

* remove unused

* added code description section

* None is converted to empty string

* integrated dataset_description

* removed unused

* refactored to an object base format

* added directory manifest generation

* added section to be removed

* added python-magic dependency

* fixed due to unit testing

* added some uit tests

* let codeclimate know its plate

* codeclimate still dose not like me

* trying to fix codeclimate

* renaming cimis to sds

* fixed broken imports

* exporter now support formatter_v2 which is the SDS format

* updated templates

* added required product_name

* connecting TO data

* duplicate uses fomratterV1

* enabled to test for v2 format

* some cleanup

* ignoring error

* refactored granting access rights

* added RRIDs to export format

* insserted new column to inputs and outputs

* added missing label

* adding v2 project

* reducting nesting

* moved to private

* removed nesting

* fixing broken feature :\

* i really do not wat this file to be checked

* trying to debug sys test

* ytuing to fix issues with system testing

* the last 4 outputs and inputs were missing

* fixed testsd for previous bug

* magic used for file description, extention comse from name

* @Crespov refactored signatures

* refactored RRID query

* fixing relative imports and moving

* changed to relative imports

* added missing submodule

* added missing requirements

* refactored to use pydantic

* refactored for simpler style

* minor refactor

* better readability

* changed to tempfile

* cahnged to tempfile

* repalced text generation with faker

* using ServiceVersion already defined

* refactored timestamp and datetime

* fixing pylint

* fixed regex for service_version

* fixed expected dateitime format

* removed rev2

* added description overwrites

* properly raise error instead of crashing without a reason

* avoids issues with tsr

* fixed relative imports

* making easier to find out missing services

* fixing pylint

Co-authored-by: Andrei Neagu <[email protected]>

.codeclimate.yml

@@ -80,3 +80,4 @@ exclude_patterns:
   - "**/pytest-simcore/"
   - "**/pytest_plugin/"
   - "**/sandbox/"
+  - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py

api/specs/webserver/openapi.yaml

@@ -168,7 +168,7 @@ paths:
   /projects/{project_id}/state:
     $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1state"
 
-  /projects/{project_id}:xport:
+  /projects/{project_id}:xport: # do not change there "export" will not work
     $ref: "./openapi-projects.yaml#/paths/~1projects~1{project_id}~1xport"
 
   /projects/{project_id}:duplicate:
@@ -236,7 +236,6 @@ paths:
   /catalog/services/{service_key}/{service_version}/outputs:match:
     $ref: "./openapi-catalog.yaml#/paths/catalog_services_service_key_service_version_outputs_match"
 
-
   # VIEWER -------------------------------------------------------------------------
   /viewers:
     $ref: "./openapi-viewer.yaml#/paths/~1viewers"

services/web/Dockerfile

@@ -73,6 +73,7 @@ ENV SC_BUILD_TARGET build
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
     build-essential \
+    libmagic-dev \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

services/web/server/requirements/_base.in

@@ -31,6 +31,8 @@ semantic_version
 aiodebug
 json2html
 parfive
+openpyxl
+python-magic
 
 pydantic[email]
 orjson

services/web/server/requirements/_base.txt

@@ -97,6 +97,8 @@ dnspython==2.0.0
     # via email-validator
 email-validator==1.1.1
     # via pydantic
+et-xmlfile==1.0.1
+    # via openpyxl
 expiringdict==1.2.1
     # via -r requirements/_base.in
 hiredis==1.1.0
@@ -150,6 +152,8 @@ openapi-core==0.12.0
     # via -r requirements/../../../../packages/service-library/requirements/_base.in
 openapi-spec-validator==0.2.9
     # via openapi-core
+openpyxl==3.0.7
+    # via -r requirements/_base.in
 orjson==3.4.8
     # via -r requirements/_base.in
 pamqp==2.3.0
@@ -178,6 +182,8 @@ pyrsistent==0.16.0
     # via jsonschema
 python-engineio==3.13.2
     # via python-socketio
+python-magic==0.4.22
+    # via -r requirements/_base.in
 python-socketio==4.6.0
     # via -r requirements/_base.in
 pytz==2020.1

services/web/server/src/simcore_service_webserver/exporter/export_import.py

@@ -1,5 +1,6 @@
 import logging
 from pathlib import Path
+from typing import Type
 
 import aiofiles
 from aiohttp import web
@@ -8,7 +9,7 @@
 from .archiving import unzip_folder, validate_osparc_import_name, zip_folder
 from .async_hashing import checksum
 from .exceptions import ExporterException
-from .formatters import BaseFormatter, FormatterV1, validate_manifest
+from .formatters import BaseFormatter, FormatterV2, validate_manifest
 
 log = logging.getLogger(__name__)
 
@@ -18,7 +19,9 @@ async def study_export(
     tmp_dir: str,
     project_id: str,
     user_id: int,
+    product_name: str,
     archive: bool = False,
+    formatter_class: Type[BaseFormatter] = FormatterV2,
 ) -> Path:
     """
     Generates a folder with all the data necessary for exporting a project.
@@ -32,9 +35,9 @@ async def study_export(
     destination.mkdir(parents=True, exist_ok=True)
 
     # The formatter will always be chosen to be the highest availabel version
-    formatter = FormatterV1(root_folder=destination)
+    formatter = formatter_class(root_folder=destination)
     await formatter.format_export_directory(
-        app=app, project_id=project_id, user_id=user_id
+        app=app, project_id=project_id, user_id=user_id, product_name=product_name
     )
 
     if archive is False:

services/web/server/src/simcore_service_webserver/exporter/formatters/__init__.py

@@ -5,10 +5,11 @@
 from .base_formatter import BaseFormatter
 from .models import ManifestFile
 from .formatter_v1 import FormatterV1
-
+from .formatter_v2 import FormatterV2
+from ..exceptions import ExporterException
 
 # maps manifest version to available formatters
-_FORMATTERS_MAPPINGS: Dict[str, BaseFormatter] = {"1": FormatterV1}
+_FORMATTERS_MAPPINGS: Dict[str, BaseFormatter] = {"1": FormatterV1, "2": FormatterV2}
 
 
 async def validate_manifest(unzipped_root_folder: Path) -> BaseFormatter:
@@ -21,5 +22,14 @@ async def validate_manifest(unzipped_root_folder: Path) -> BaseFormatter:
         root_dir=unzipped_root_folder
     )
 
+    if manifest_from_file.version not in _FORMATTERS_MAPPINGS:
+        raise ExporterException(
+            (
+                f"Version {manifest_from_file.version} is not supported by this deployment. "
+                "The project you are trying to import might be exported in a newer version, "
+                "or this deployment is using an older version."
+            )
+        )
+
     formatter_cls: BaseFormatter = _FORMATTERS_MAPPINGS[manifest_from_file.version]
     return formatter_cls(root_folder=unzipped_root_folder)

services/web/server/src/simcore_service_webserver/exporter/formatters/base_formatter.py

@@ -3,14 +3,18 @@
 from abc import abstractmethod
 from pathlib import Path
 
+from aiohttp import web
+
 
 class BaseFormatter:
     def __init__(self, version: str, root_folder: Path):
         self.version: str = version
         self.root_folder: Path = root_folder
 
     @abstractmethod
-    async def format_export_directory(self, **kwargs) -> None:
+    async def format_export_directory(
+        self, app: web.Application, project_id: str, user_id: int, **kwargs
+    ) -> None:
         """Creates the output format given the current version
         and saves all data to the relative path."""
 

services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py

@@ -5,7 +5,7 @@
 from collections import deque
 from itertools import chain
 from pathlib import Path
-from typing import Deque, Dict, List, Tuple
+from typing import Deque, Dict, List, Tuple, Optional
 
 import aiofiles
 from aiohttp import ClientSession, ClientTimeout, web
@@ -124,7 +124,12 @@ async def extract_download_links(
 
 
 async def generate_directory_contents(
-    app: web.Application, root_folder: Path, project_id: str, user_id: int, version: str
+    app: web.Application,
+    root_folder: Path,
+    manifest_root_folder: Optional[Path],
+    project_id: str,
+    user_id: int,
+    version: str,
 ) -> None:
     try:
         project_data = await get_project_for_user(
@@ -151,7 +156,9 @@ async def generate_directory_contents(
         version=version,
         attachments=[str(x.store_path) for x in download_links],
     )
-    await ManifestFile.model_to_file(root_dir=root_folder, **manifest_params)
+    await ManifestFile.model_to_file(
+        root_dir=manifest_root_folder or root_folder, **manifest_params
+    )
 
     # store project data on disk
     await ProjectFile.model_to_file(root_dir=root_folder, **project_data)
@@ -338,7 +345,10 @@ async def _upload_files_to_storage(
 
 
 async def import_files_and_validate_project(
-    app: web.Application, user_id: int, root_folder: Path
+    app: web.Application,
+    user_id: int,
+    root_folder: Path,
+    manifest_root_folder: Optional[Path],
 ) -> str:
     project_file = await ProjectFile.model_from_file(root_dir=root_folder)
     shuffled_data: ShuffledData = project_file.get_shuffled_uuids()
@@ -353,7 +363,9 @@ async def import_files_and_validate_project(
     log.debug("Shuffled project data: %s", shuffled_project_file)
 
     # NOTE: it is not necessary to apply data shuffling to the manifest
-    manifest_file = await ManifestFile.model_from_file(root_dir=root_folder)
+    manifest_file = await ManifestFile.model_from_file(
+        root_dir=manifest_root_folder or root_folder
+    )
 
     user: Dict = await get_user(app=app, user_id=user_id)
 
@@ -414,17 +426,19 @@ async def import_files_and_validate_project(
 
 
 class FormatterV1(BaseFormatter):
-    def __init__(self, root_folder: Path):
-        super().__init__(version="1", root_folder=root_folder)
+    def __init__(self, root_folder: Path, version: str = "1"):
+        super().__init__(version=version, root_folder=root_folder)
 
-    async def format_export_directory(self, **kwargs) -> None:
-        app: web.Application = kwargs["app"]
-        project_id: str = kwargs["project_id"]
-        user_id: int = kwargs["user_id"]
+    async def format_export_directory(
+        self, app: web.Application, project_id: str, user_id: int, **kwargs
+    ) -> None:
+        # injected by Formatter_V2
+        manifest_root_folder: Optional[Path] = kwargs.get("manifest_root_folder")
 
         await generate_directory_contents(
             app=app,
             root_folder=self.root_folder,
+            manifest_root_folder=manifest_root_folder,
             project_id=project_id,
             user_id=user_id,
             version=self.version,
@@ -433,7 +447,12 @@ async def format_export_directory(self, **kwargs) -> None:
     async def validate_and_import_directory(self, **kwargs) -> str:
         app: web.Application = kwargs["app"]
         user_id: int = kwargs["user_id"]
+        # injected by Formatter_V2
+        manifest_root_folder: Optional[Path] = kwargs.get("manifest_root_folder")
 
         return await import_files_and_validate_project(
-            app=app, user_id=user_id, root_folder=self.root_folder
+            app=app,
+            user_id=user_id,
+            root_folder=self.root_folder,
+            manifest_root_folder=manifest_root_folder,
         )