Nodeports/add file checksum (#2011)

- ensure storage is in debug mode when in devel
- nodeports v2

.coveragerc

@@ -1,6 +1,27 @@
 [run]
 branch = True
-omit = 
+omit =
     */tests/*
     */generated_code/*
 parallel = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+show_missing = True

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -159,6 +159,8 @@ properties:
                         type: string
                       label:
                         type: string
+                      eTag:
+                        type: string
                   - type: object
                     additionalProperties: false
                     required:
@@ -221,6 +223,8 @@ properties:
                         type: string
                       label:
                         type: string
+                      eTag:
+                        type: string
                   - type: object
                     additionalProperties: false
                     required:

api/specs/common/schemas/project-v0.0.1.json

@@ -210,6 +210,9 @@
                         },
                         "label": {
                           "type": "string"
+                        },
+                        "eTag": {
+                          "type": "string"
                         }
                       }
                     },
@@ -302,6 +305,9 @@
                         },
                         "label": {
                           "type": "string"
+                        },
+                        "eTag": {
+                          "type": "string"
                         }
                       }
                     },
@@ -344,7 +350,10 @@
               ]
             },
             "parent": {
-              "type": [ "null", "string" ],
+              "type": [
+                "null",
+                "string"
+              ],
               "format": "uuid",
               "description": "Parent's (group-nodes') node ID s.",
               "examples": [

packages/models-library/src/models_library/projects_nodes_io.py

@@ -9,7 +9,6 @@
 
 from .services import PROPERTY_KEY_RE
 
-
 NodeID = UUID
 
 # Pydantic does not support exporting a jsonschema with Dict keys being something else than a str
@@ -54,12 +53,18 @@ class BaseFileLink(BaseModel):
     )
     path: str = Field(
         ...,
+        regex=r"^.+$",
         description="The path to the file in the storage provider domain",
         example=[
             "N:package:b05739ef-260c-4038-b47d-0240d04b0599",
             "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt",
         ],
     )
+    e_tag: Optional[str] = Field(
+        None,
+        description="Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box).",
+        alias="eTag",
+    )
 
     class Config:
         extra = Extra.forbid

packages/models-library/src/models_library/services.py

@@ -6,7 +6,18 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, constr, validator
+from pydantic import (
+    BaseModel,
+    EmailStr,
+    Extra,
+    Field,
+    HttpUrl,
+    StrictBool,
+    StrictFloat,
+    StrictInt,
+    constr,
+    validator,
+)
 from pydantic.types import PositiveInt
 
 from .basic_regex import VERSION_RE
@@ -150,7 +161,7 @@ class ServiceProperty(BaseModel):
         description="Place the data associated with the named keys in files",
         examples=[{"dir/input1.txt": "key_1", "dir33/input2.txt": "key2"}],
     )
-    default_value: Optional[Union[str, float, bool, int]] = Field(
+    default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field(
         None, alias="defaultValue", examples=["Dog", True]
     )
 

packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py

@@ -4,9 +4,10 @@
 from aioresponses import aioresponses
 from aioresponses.core import CallbackResult
 from models_library.projects_state import RunningState
+from yarl import URL
 
 
-def creation_cb(url, **kwargs):
+def creation_cb(url, **kwargs) -> CallbackResult:
 
     assert "json" in kwargs, f"missing body in call to {url}"
     body = kwargs["json"]
@@ -24,7 +25,7 @@ def creation_cb(url, **kwargs):
 
 
 @pytest.fixture
-async def director_v2_subsystem_mock() -> aioresponses:
+async def director_v2_service_mock() -> aioresponses:
 
     """uses aioresponses to mock all calls of an aiohttpclient
     WARNING: any request done through the client will go through aioresponses. It is
@@ -63,3 +64,38 @@ async def director_v2_subsystem_mock() -> aioresponses:
         mock.delete(delete_computation_pattern, status=204, repeat=True)
 
         yield mock
+
+
+@pytest.fixture
+async def storage_v0_service_mock() -> aioresponses:
+
+    """uses aioresponses to mock all calls of an aiohttpclient
+    WARNING: any request done through the client will go through aioresponses. It is
+    unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough.
+    Other than that it seems to behave nicely
+    """
+    PASSTHROUGH_REQUESTS_PREFIXES = ["http://127.0.0.1", "ws://"]
+
+    def get_download_link_cb(url: URL, **kwargs) -> CallbackResult:
+        file_id = url.path.rsplit("/files/")[1]
+
+        return CallbackResult(
+            status=200, payload={"data": {"link": f"file://{file_id}"}}
+        )
+
+    get_download_link_pattern = re.compile(
+        r"^http://[a-z\-_]*storage:[0-9]+/v0/locations/[0-9]+/files/.+$"
+    )
+
+    get_locations_link_pattern = re.compile(
+        r"^http://[a-z\-_]*storage:[0-9]+/v0/locations.*$"
+    )
+
+    with aioresponses(passthrough=PASSTHROUGH_REQUESTS_PREFIXES) as mock:
+        mock.get(get_download_link_pattern, callback=get_download_link_cb, repeat=True)
+        mock.get(
+            get_locations_link_pattern,
+            status=200,
+            payload={"data": [{"name": "simcore.s3", "id": "0"}]},
+        )
+        yield mock

packages/simcore-sdk/requirements/_base.in

@@ -12,8 +12,9 @@ aiohttp
 aiopg[sa]
 networkx
 psycopg2-binary
-pydantic
+pydantic[email]
 tenacity
+tqdm
 trafaret-config
 
 attrs

packages/simcore-sdk/requirements/_base.txt

@@ -40,6 +40,7 @@ six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-
 sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../requirements/constraints.txt, -r requirements/../../../packages/postgres-database/requirements/_base.in, -r requirements/../../../packages/service-library/requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in
+tqdm==4.54.1              # via -r requirements/_base.in
 trafaret-config==2.0.2    # via -r requirements/_base.in
 trafaret==2.1.0           # via -r requirements/../../../packages/service-library/requirements/_base.in, trafaret-config
 typing-extensions==3.7.4.3  # via aiohttp, yarl

packages/simcore-sdk/requirements/_test.in

@@ -20,6 +20,7 @@ pytest-sugar
 pytest-xdist
 
 # mockups/fixtures
+alembic
 aioresponses
 requests
 docker

packages/simcore-sdk/requirements/_test.txt

@@ -6,6 +6,7 @@
 #
 aiohttp==3.7.3            # via -c requirements/_base.txt, aioresponses, pytest-aiohttp
 aioresponses==0.7.1       # via -r requirements/_test.in
+alembic==1.4.3            # via -r requirements/_test.in
 apipkg==1.5               # via execnet
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via -c requirements/_base.txt, aiohttp
@@ -24,11 +25,14 @@ importlib-metadata==3.1.1  # via -c requirements/_base.txt, pluggy, pytest
 iniconfig==1.1.1          # via pytest
 isort==5.6.4              # via pylint
 lazy-object-proxy==1.4.3  # via -c requirements/_base.txt, astroid
+mako==1.1.3               # via alembic
+markupsafe==1.1.1         # via mako
 mccabe==0.6.1             # via pylint
 multidict==5.1.0          # via -c requirements/_base.txt, aiohttp, yarl
 packaging==20.7           # via pytest, pytest-sugar
 pluggy==0.13.1            # via pytest
 pprintpp==0.4.0           # via pytest-icdiff
+psycopg2-binary==2.8.6    # via -c requirements/_base.txt, sqlalchemy
 py==1.9.0                 # via pytest, pytest-forked
 pylint==2.6.0             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
@@ -42,9 +46,12 @@ pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
 pytest-xdist==2.1.0       # via -r requirements/_test.in
 pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist
+python-dateutil==2.8.1    # via -c requirements/_base.txt, alembic
 python-dotenv==0.15.0     # via -r requirements/_test.in
+python-editor==1.0.4      # via alembic
 requests==2.25.0          # via -r requirements/_test.in, coveralls, docker
-six==1.15.0               # via -c requirements/_base.txt, astroid, docker, websocket-client
+six==1.15.0               # via -c requirements/_base.txt, astroid, docker, python-dateutil, websocket-client
+sqlalchemy[postgresql_psycopg2binary]==1.3.20  # via -c requirements/_base.txt, alembic
 termcolor==1.1.0          # via pytest-sugar
 toml==0.10.2              # via pylint, pytest
 typed-ast==1.4.1          # via astroid

packages/simcore-sdk/setup.cfg

@@ -0,0 +1,19 @@
+[bumpversion]
+current_version = 0.3.0
+commit = True
+tag = False
+
+[bumpversion:file:setup.py]
+search = version='{current_version}'
+replace = version='{new_version}'
+
+[bumpversion:file:src/simcore_sdk/__init__.py]
+search = __version__ = '{current_version}'
+replace = __version__ = '{new_version}'
+
+[bdist_wheel]
+universal = 1
+
+[aliases]
+# Define setup.py command aliases here
+test = pytest

packages/simcore-sdk/setup.py

@@ -24,7 +24,7 @@ def read_reqs(reqs_path: Path):
 
 setup(
     name="simcore-sdk",
-    version="0.2.0",
+    version="0.3.0",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     python_requires=">=3.6",

packages/simcore-sdk/src/simcore_sdk/__init__.py

@@ -0,0 +1,5 @@
+""" osparc's simcore-sdk library
+
+"""
+
+__version__ = "0.3.0"

packages/simcore-sdk/src/simcore_sdk/config/db.py

@@ -5,21 +5,22 @@
 
 import trafaret as T
 
-CONFIG_SCHEMA = T.Dict({
-    "database": T.String(),
-    "user": T.String(),
-    "password": T.String(),
-    T.Key("minsize", default=1 ,optional=True): T.ToInt(),
-    T.Key("maxsize", default=4, optional=True): T.ToInt(),
-    "host": T.Or( T.String, T.Null),
-    "port": T.Or( T.ToInt, T.Null),
-    "endpoint": T.Or( T.String, T.Null)
-})
+CONFIG_SCHEMA = T.Dict(
+    {
+        "database": T.String(),
+        "user": T.String(),
+        "password": T.String(),
+        T.Key("minsize", default=1, optional=True): T.ToInt(),
+        T.Key("maxsize", default=4, optional=True): T.ToInt(),
+        "host": T.Or(T.String, T.Null),
+        "port": T.Or(T.ToInt, T.Null),
+        "endpoint": T.Or(T.String, T.Null),
+    }
+)
 
 
 # TODO: deprecate!
-class Config():
-
+class Config:
     def __init__(self):
         # TODO: uniform config classes . see server.config file
         POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432")
@@ -31,8 +32,9 @@ def __init__(self):
         self._pwd = POSTGRES_PW
         self._url = POSTGRES_URL
         self._db = POSTGRES_DB
-        self._endpoint = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(
-            user=self._user, pw=self._pwd, url=self._url, db=self._db)
+        self._endpoint = "postgresql+psycopg2://{user}:{pw}@{url}/{db}".format(
+            user=self._user, pw=self._pwd, url=self._url, db=self._db
+        )
 
     @property
     def endpoint(self):

packages/simcore-sdk/src/simcore_sdk/models/__init__.py

@@ -1,8 +1,7 @@
 from . import pipeline_models
+from .base import metadata
+
 # Add here new models
 
-from .base import metadata
 
-__all__ = (
-    'metadata'
-)
+__all__ = "metadata"

packages/simcore-sdk/src/simcore_sdk/models/base.py

@@ -1,5 +1,3 @@
 from simcore_postgres_database.models.base import metadata
 
-__all__ = [
-    "metadata"
-]
+__all__ = ["metadata"]

packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py

@@ -1 +1 @@
-from . import data_manager
+from . import data_manager

packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import warnings
 
 from . import config as node_config
 from . import exceptions
@@ -10,3 +11,6 @@
 log = logging.getLogger(__name__)
 log.addHandler(logging.NullHandler())
 
+warnings.warn(
+    "node_ports is deprecated, use node_ports_v2 instead", category=DeprecationWarning
+)