feat: added renku service with cache and datasets

.env

@@ -0,0 +1,4 @@
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_DATABASE=0
+REDIS_PASSWORD=

.gitignore

@@ -73,3 +73,5 @@ target/
 renku-*.bottle.json
 renku-*.bottle.tar.gz
 renku.rb
+
+.env

Dockerfile → Dockerfile.cli

@@ -1,4 +1,4 @@
-FROM python:3.6-alpine as base
+FROM python:3.7-alpine as base
 
 RUN apk add --no-cache git && \
     pip install --no-cache --upgrade pip

Dockerfile.svc

@@ -0,0 +1,18 @@
+FROM python:3.7-alpine
+
+RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git && \
+    pip install --no-cache --upgrade pip setuptools pipenv requirements-builder
+
+RUN apk add --no-cache --allow-untrusted \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \
+    --repository http://nl.alpinelinux.org/alpine/edge/community \
+    git-lfs && \
+    git lfs install
+
+COPY . /code/renku
+WORKDIR /code/renku
+RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn
+
+
+ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"]

MANIFEST.in

@@ -17,15 +17,17 @@
 # limitations under the License.
 
 # Check manifest will not automatically add these two files:
+include renku/service/.env-example
 include .dockerignore
 include .editorconfig
 include .tx/config
 include *.md
 prune docs/_build
 recursive-include renku *.po *.pot *.mo
-
+recursive-include renku *.py
 # added by check_manifest.py
 include *.py
+include *.yml
 include *.rst
 include *.sh
 include *.txt
@@ -39,6 +41,7 @@ include babel.ini
 include brew.py
 include pytest.ini
 include snap/snapcraft.yaml
+recursive-include renku *.json
 recursive-include .github CODEOWNERS
 recursive-include .travis *.sh
 recursive-include docs *.bat
@@ -59,3 +62,4 @@ recursive-include renku *.yml
 recursive-include renku Dockerfile
 recursive-include tests *.py *.gz *.yml
 prune .github
+prune .env

Makefile

@@ -69,3 +69,6 @@ brew-commit-bottle: *.bottle.json
 
 brew-release:
 	open "https://github.com/SwissDataScienceCenter/renku-python/releases/new?tag=v$(shell brew info --json=v1 renku | jq -r '.[0].versions.stable')"
+
+service-container:
+	docker build -f Dockerfile.svc -t renku-svc .

conftest.py

@@ -25,13 +25,19 @@
 import tempfile
 import time
 import urllib
+import uuid
 from pathlib import Path
 
+import fakeredis
 import pytest
 import responses
 import yaml
 from click.testing import CliRunner
 
+from renku.core.utils.contexts import chdir
+from renku.service.cache import ServiceCache
+from renku.service.entrypoint import create_app
+
 
 @pytest.fixture(scope='module')
 def renku_path(tmpdir_factory):
@@ -510,3 +516,86 @@ def remote_project(data_repository, directory_tree):
         assert 0 == result.exit_code
 
         yield runner, project_path
+
+
+@pytest.fixture(scope='function')
+def dummy_datapack():
+    """Creates dummy data folder."""
+    temp_dir = tempfile.TemporaryDirectory()
+
+    data_file_txt = Path(temp_dir.name) / Path('file.txt')
+    data_file_txt.write_text('my awesome data')
+
+    data_file_csv = Path(temp_dir.name) / Path('file.csv')
+    data_file_csv.write_text('more,awesome,data')
+
+    yield temp_dir
+
+
+@pytest.fixture(scope='function')
+def datapack_zip(dummy_datapack):
+    """Returns dummy data folder as a zip archive."""
+    workspace_dir = tempfile.TemporaryDirectory()
+    with chdir(workspace_dir.name):
+        shutil.make_archive('datapack', 'zip', dummy_datapack.name)
+
+    yield Path(workspace_dir.name) / 'datapack.zip'
+
+
+@pytest.fixture(scope='function')
+def datapack_tar(dummy_datapack):
+    """Returns dummy data folder as a tar archive."""
+    workspace_dir = tempfile.TemporaryDirectory()
+    with chdir(workspace_dir.name):
+        shutil.make_archive('datapack', 'tar', dummy_datapack.name)
+
+    yield Path(workspace_dir.name) / 'datapack.tar'
+
+
+@pytest.fixture(scope='function')
+def mock_redis(monkeypatch):
+    """Monkey patch service cache with mocked redis."""
+    with monkeypatch.context() as m:
+        m.setattr(ServiceCache, 'cache', fakeredis.FakeRedis())
+        yield
+
+
+@pytest.fixture(scope='function')
+def svc_client(mock_redis):
+    """Renku service client."""
+    flask_app = create_app()
+
+    testing_client = flask_app.test_client()
+    testing_client.testing = True
+
+    ctx = flask_app.app_context()
+    ctx.push()
+
+    yield testing_client
+
+    ctx.pop()
+
+
+@pytest.fixture(scope='function')
+def svc_client_with_repo(svc_client, mock_redis):
+    """Renku service remote repository."""
+    remote_url = 'https://renkulab.io/gitlab/contact/integration-tests.git'
+    headers = {'Authorization': 'Bearer b4b4de0eda0f471ab82702bd5c367fa7'}
+
+    params = {
+        'git_url': remote_url,
+        'git_username': 'contact',
+        'git_access_token': 'EcfPJvEqjJepyu6XyqKZ',
+    }
+
+    response = svc_client.post(
+        '/cache/project-clone', data=params, headers=headers
+    )
+
+    assert response
+    assert 'result' in response.json
+    assert 'error' not in response.json
+    project_id = response.json['result']['project_id']
+    assert isinstance(uuid.UUID(project_id), uuid.UUID)
+
+    yield svc_client, headers, project_id

docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  redis:
+    image: redis:5.0.3-alpine
+    ports:
+      - "6379:6379"
+
+  renku-svc:
+    image: renku-svc:latest
+    env_file: .env
+    ports:
+      - "8080:8080"

renku/core/commands/client.py

@@ -25,6 +25,8 @@
 import yaml
 
 from renku.core.management import LocalClient
+from renku.core.management.config import RENKU_HOME
+from renku.core.management.repository import default_path
 
 from .git import get_git_isolation
 
@@ -63,8 +65,17 @@ def pass_local_client(
         )
 
     def new_func(*args, **kwargs):
-        ctx = click.get_current_context()
-        client = ctx.ensure_object(LocalClient)
+        ctx = click.get_current_context(silent=True)
+        if not ctx:
+            client = LocalClient(
+                path=default_path(),
+                renku_home=RENKU_HOME,
+                use_external_storage=True,
+            )
+            ctx = click.Context(click.Command(method))
+        else:
+            client = ctx.ensure_object(LocalClient)
+
         stack = contextlib.ExitStack()
 
         # Handle --isolation option:
@@ -85,8 +96,11 @@ def new_func(*args, **kwargs):
         if lock or (lock is None and commit):
             stack.enter_context(client.lock)
 
-        with stack:
-            result = ctx.invoke(method, client, *args, **kwargs)
+        result = None
+        if ctx:
+            with stack:
+                result = ctx.invoke(method, client, *args, **kwargs)
+
         return result
 
     return functools.update_wrapper(new_func, method)

renku/core/commands/dataset.py

@@ -143,9 +143,12 @@ def add_file(
     destination='',
     ref=None,
     with_metadata=None,
-    urlscontext=contextlib.nullcontext
+    urlscontext=contextlib.nullcontext,
+    use_external_storage=False
 ):
     """Add data file to a dataset."""
+    client.use_external_storage = use_external_storage
+
     add_to_dataset(
         client, urls, name, link, force, create, sources, destination, ref,
         with_metadata, urlscontext

renku/core/management/repository.py

@@ -47,13 +47,18 @@ def default_path():
         return '.'
 
 
+def path_converter(path):
+    """Converter for path in PathMixin."""
+    return Path(path).resolve()
+
+
 @attr.s
 class PathMixin:
     """Define a default path attribute."""
 
     path = attr.ib(
         default=default_path,
-        converter=lambda arg: Path(arg).resolve().absolute(),
+        converter=path_converter,
     )
 
     @path.validator

renku/core/utils/contexts.py

@@ -26,6 +26,9 @@
 @contextlib.contextmanager
 def chdir(path):
     """Change the current working directory."""
+    if isinstance(path, Path):
+        path = str(path)
+
     cwd = os.getcwd()
     os.chdir(path)
     try:

renku/service/.env-example

@@ -0,0 +1,6 @@
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_DATABASE=0
+REDIS_PASSWORD=
+
+CACHE_DIR=

renku/service/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service."""

renku/service/cache/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service cache management for files."""
+from renku.service.cache.files import FileManagementCache
+from renku.service.cache.projects import ProjectManagementCache
+
+
+class ServiceCache(FileManagementCache, ProjectManagementCache):
+    """Service cache manager."""
+
+    pass

renku/service/cache/base.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service cache management."""
+import json
+
+import redis
+from redis import RedisError
+
+from renku.service.cache.config import REDIS_DATABASE, REDIS_HOST, \
+    REDIS_PASSWORD, REDIS_PORT
+
+
+class BaseCache:
+    """Cache management."""
+
+    cache = redis.Redis(
+        host=REDIS_HOST,
+        port=REDIS_PORT,
+        db=REDIS_DATABASE,
+        password=REDIS_PASSWORD
+    )
+
+    def set_record(self, name, key, value):
+        """Insert a record to hash set."""
+        if isinstance(value, dict):
+            value = json.dumps(value)
+
+        self.cache.hset(name, key, value)
+
+    def invalidate_key(self, name, key):
+        """Invalidate cache `key` in users hash set."""
+        try:
+            self.cache.hdel(name, key)
+        except RedisError:
+            pass
+
+    def get_record(self, name, key):
+        """Return record values from hash set."""
+        result = self.cache.hget(name, key)
+        if result:
+            return json.loads(result)
+
+    def get_all_records(self, name):
+        """Return all record values from hash set."""
+        return [
+            json.loads(record) for record in self.cache.hgetall(name).values()
+        ]