diff --git a/Dockerfile b/Dockerfile.cli similarity index 96% rename from Dockerfile rename to Dockerfile.cli index 770a6e39ee..6e63b77185 100644 --- a/Dockerfile +++ b/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM python:3.6-alpine as base +FROM python:3.7-alpine as base RUN apk add --no-cache git && \ pip install --no-cache --upgrade pip diff --git a/Dockerfile.svc b/Dockerfile.svc new file mode 100644 index 0000000000..ab9f4d06e7 --- /dev/null +++ b/Dockerfile.svc @@ -0,0 +1,18 @@ +FROM python:3.7-alpine + +RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git && \ + pip install --no-cache --upgrade pip setuptools pipenv requirements-builder + +RUN apk add --no-cache --allow-untrusted \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \ + --repository http://nl.alpinelinux.org/alpine/edge/community \ + git-lfs && \ + git lfs install + +COPY . /code/renku +WORKDIR /code/renku +RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn + + +ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"] diff --git a/MANIFEST.in b/MANIFEST.in index ab3e9b5e3c..951be9e171 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -39,6 +39,7 @@ include babel.ini include brew.py include pytest.ini include snap/snapcraft.yaml +recursive-include renku *.json recursive-include .github CODEOWNERS recursive-include .travis *.sh recursive-include docs *.bat diff --git a/Pipfile.lock b/Pipfile.lock index 936bc038a6..368c8b5734 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a3e161caf52b39ed8aa1de4a3306a163ca1043dfa219203a4cc1b11463c9007b" + "sha256": "c61855aa6c4438d0e2af40a860510eadc567849b6b0e0fba0fa1ba76663a89cb" }, "pipfile-spec": 6, "requires": { @@ -30,6 +30,25 @@ ], "version": "==1.5" }, + "apispec": { + "extras": [ + "yaml" + ], + "hashes": [ + "sha256:5fdaa1173b32515cc83f9d413a49a6c37fafc2b87f6b40e95923d3e85f0942c5", + "sha256:9e88c51517a6515612e818459f61c1bc06c00f2313e5187828bdbabaa7461473" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "apispec-webframeworks": { + "hashes": [ + "sha256:02fb79a7e37bc4e71ad21f6a9ddfbfc8e919eede7ef685d35d2d8549c2d0282d", + "sha256:89502de27f87e10766a62c9caf2ce4d33abce3acda91ae50abb3ef4937763b59" + ], + "index": "pypi", + "version": "==0.5.0" + }, "appdirs": { "hashes": [ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", @@ -37,13 +56,6 @@ ], "version": "==1.4.3" }, - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, "atomicwrites": { "hashes": [ "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", @@ -53,10 +65,10 @@ }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "avro-cwl": { "hashes": [ @@ -94,36 +106,38 @@ }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", + "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", + "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", + "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", + "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", + "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", + "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", + "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", + "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", + "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", + "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", + "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", + "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", + "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", + "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", + "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", + "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", + "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", + "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", + "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", + "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", + "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", + "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", + "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", + "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", + "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", + "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", + "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", + "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", + "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" + ], + "version": "==1.13.1" }, "chardet": { "hashes": [ @@ -134,10 +148,10 @@ }, "check-manifest": { "hashes": [ - "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", - "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" + "sha256:42de6eaab4ed149e60c9b367ada54f01a3b1e4d6846784f9b9710e770ff5572c", + "sha256:78dd077f2c70dbac7cfcc9d12cbd423914e787ea4b5631de45aecd25b524e8e3" ], - "version": "==0.39" + "version": "==0.40" }, "click": { "hashes": [ @@ -148,9 +162,9 @@ }, "click-completion": { "hashes": [ - "sha256:78072eecd5e25ea0d25ceaf99cd5f22aa2667d67231ae0819deab9b1ff3456fb" + "sha256:5bf816b81367e638a190b6e91b50779007d14301b3f9f3145d68e3cade7bce86" ], - "version": "==0.5.1" + "version": "==0.5.2" }, "coverage": { "hashes": [ @@ -191,24 +205,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" - ], - "version": "==2.7" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" }, "cwlref-runner": { "hashes": [ @@ -274,6 +293,20 @@ ], "version": "==3.7.8" }, + "flask": { + "hashes": [ + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + ], + "version": "==1.1.1" + }, + "flask-swagger-ui": { + "hashes": [ + "sha256:3282c770764c8053360f33b2fc120e1d169ecca2138537d0e6e1135b1f9d4ff2" + ], + "index": "pypi", + "version": "==3.20.9" + }, "freezegun": { "hashes": [ "sha256:2a4d9c8cd3c04a201e20c313caf8b6338f1cfa4cda43f46a94cc4a9fd13ea5e7", @@ -298,10 +331,10 @@ }, "gitpython": { "hashes": [ - "sha256:631263cc670aa56ce3d3c414cf0fe2e840f2e913514b138ea28d88a477bbcd21", - "sha256:6e97b9f0954807f30c2dd8e3165731ed6c477a1b365f194b69d81d7940a08332" + "sha256:3237caca1139d0a7aa072f6735f5fd2520de52195e0fa1d8b83a9b212a2498b2", + "sha256:a7d6bef0775f66ba47f25911d285bcd692ce9053837ff48a120c2b8cf3a71389" ], - "version": "==3.0.3" + "version": "==3.0.4" }, "idna": { "hashes": [ @@ -340,6 +373,13 @@ ], "version": "==4.3.4" }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, "jinja2": { "hashes": [ "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", @@ -437,9 +477,10 @@ }, "mypy-extensions": { "hashes": [ - "sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458" + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" ], - "version": "==0.4.2" + "version": "==0.4.3" }, "ndg-httpsclient": { "hashes": [ @@ -451,9 +492,10 @@ }, "networkx": { "hashes": [ - "sha256:8311ddef63cf5c5c5e7c1d0212dd141d9a1fe3f474915281b73597ed5f1d4e3d" + "sha256:cdfbf698749a5014bf2ed9db4a07a5295df1d3a53bf80bf3cbd61edf9df05fa1", + "sha256:f8f4ff0b6f96e4f9b16af6b84622597b5334bf9cae8cf9b2e42e7985d5c95c64" ], - "version": "==2.3" + "version": "==2.4" }, "packaging": { "hashes": [ @@ -552,6 +594,14 @@ ], "version": "==2.4.2" }, + "pyjwt": { + "hashes": [ + "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", + "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" + ], + "index": "pypi", + "version": "==1.7.1" + }, "pyld": { "hashes": [ "sha256:ce6d9cd065fb3a390ec65e665dcb3655ed2aa07431d98e201ea3bc99f56a8bfb" @@ -753,10 +803,10 @@ }, "sentry-sdk": { "hashes": [ - "sha256:15e51e74b924180c98bcd636cb4634945b0a99a124d50b433c3a9dc6a582e8db", - "sha256:1d6a2ee908ec6d8f96c27d78bc39e203df4d586d287c233140af7d8d1aca108a" + "sha256:7d8668f082cb1eb9bf1e0d3f8f9bd5796d05d927c1197af226d044ed32b9815f", + "sha256:ff14935cc3053de0650128f124c36f34a4be120b8cc522c149f5cba342c1fd05" ], - "version": "==0.12.3" + "version": "==0.13.0" }, "setuptools-scm": { "hashes": [ diff --git a/conftest.py b/conftest.py index 117d26cc90..a108851c43 100644 --- a/conftest.py +++ b/conftest.py @@ -30,6 +30,8 @@ import responses from click.testing import CliRunner +from renku.service.entrypoint import create_app + @pytest.fixture(scope='module') def renku_path(tmpdir_factory): @@ -474,3 +476,40 @@ def sleep_after(): import time yield time.sleep(0.5) + + +@pytest.fixture(scope='module') +def svc_client(): + """Renku service client.""" + flask_app = create_app() + + testing_client = flask_app.test_client() + testing_client.testing = True + + ctx = flask_app.app_context() + ctx.push() + + yield testing_client + + ctx.pop() + + +@pytest.fixture(scope='function') +def svc_client_with_repo(svc_client): + """Renku service remote repository.""" + access_token = 'contact:EcfPJvEqjJepyu6XyqKZ' + remote_url = 'https://{0}@renkulab.io/gitlab/contact/integration-tests.git' + headers = {'Authorization': 'Bearer b4b4de0eda0f471ab82702bd5c367fa7'} + + params = {'git_url': remote_url.format(access_token), 'force': 1} + + response = svc_client.get( + '/cache/project-clone', query_string=params, headers=headers + ) + + assert response + assert 'result' in response.json + assert 'error' not in response.json + assert 'integration-tests' == response.json['result']['project_id'] + + yield svc_client, headers diff --git a/renku/core/commands/client.py b/renku/core/commands/client.py index 12d103fae0..f9eec0d0e2 100644 --- a/renku/core/commands/client.py +++ b/renku/core/commands/client.py @@ -25,6 +25,8 @@ import yaml from renku.core.management import LocalClient +from renku.core.management.config import RENKU_HOME +from renku.core.management.repository import default_path from .git import get_git_isolation @@ -63,8 +65,17 @@ def pass_local_client( ) def new_func(*args, **kwargs): - ctx = click.get_current_context() - client = ctx.ensure_object(LocalClient) + ctx = click.get_current_context(silent=True) + if not ctx: + client = LocalClient( + path=default_path(), + renku_home=RENKU_HOME, + use_external_storage=True, + ) + ctx = click.Context(click.Command(method)) + else: + client = ctx.ensure_object(LocalClient) + stack = contextlib.ExitStack() # Handle --isolation option: @@ -85,8 +96,11 @@ def new_func(*args, **kwargs): if lock or (lock is None and commit): stack.enter_context(client.lock) - with stack: - result = ctx.invoke(method, client, *args, **kwargs) + result = None + if ctx: + with stack: + result = ctx.invoke(method, client, *args, **kwargs) + return result return functools.update_wrapper(new_func, method) diff --git a/renku/core/commands/dataset.py b/renku/core/commands/dataset.py index 318ae7ae1c..8d95a2bef8 100644 --- a/renku/core/commands/dataset.py +++ b/renku/core/commands/dataset.py @@ -106,7 +106,7 @@ def create_dataset(client, name, handle_duplicate_fn=None): :raises: ``renku.core.errors.ParameterError`` """ existing = client.load_dataset(name=name) - if (not existing or handle_duplicate_fn and handle_duplicate_fn(existing)): + if not existing or handle_duplicate_fn and handle_duplicate_fn(existing): with client.with_dataset(name=name) as dataset: creator = Creator.from_git(client.repo) if creator not in dataset.creator: @@ -143,9 +143,12 @@ def add_file( sources=(), destination='', with_metadata=None, - urlscontext=contextlib.nullcontext + urlscontext=contextlib.nullcontext, + use_external_storage=True ): """Add data file to a dataset.""" + client.use_external_storage = use_external_storage + add_to_dataset( client, urls, name, link, force, sources, destination, with_metadata, urlscontext diff --git a/renku/core/management/datasets.py b/renku/core/management/datasets.py index f011e235d0..6af377bd38 100644 --- a/renku/core/management/datasets.py +++ b/renku/core/management/datasets.py @@ -296,7 +296,8 @@ def _add_from_url(self, dataset, dataset_path, url, link, destination): mode = dst.stat().st_mode & 0o777 dst.chmod(mode & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) - self.track_paths_in_storage(str(dst.relative_to(self.path))) + if self.has_external_storage: + self.track_paths_in_storage(str(dst.relative_to(self.path))) return [{ 'path': dst.relative_to(self.path), diff --git a/renku/core/management/repository.py b/renku/core/management/repository.py index fec9d25b7c..998c42b0af 100644 --- a/renku/core/management/repository.py +++ b/renku/core/management/repository.py @@ -47,13 +47,18 @@ def default_path(): return '.' +def path_converter(path): + """Converter for path in PathMixin.""" + return Path(path).resolve() + + @attr.s class PathMixin: """Define a default path attribute.""" path = attr.ib( default=default_path, - converter=lambda arg: Path(arg).resolve().absolute(), + converter=path_converter, ) @path.validator diff --git a/renku/service/Dockerfile b/renku/service/Dockerfile new file mode 100644 index 0000000000..d697d06b90 --- /dev/null +++ b/renku/service/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.6 +ADD . /app +WORKDIR /app +RUN pip install -e . +EXPOSE 8000 +CMD ["gunicorn", "-b", "0.0.0.0:8000", "app"] \ No newline at end of file diff --git a/renku/service/__init__.py b/renku/service/__init__.py new file mode 100644 index 0000000000..1928b35350 --- /dev/null +++ b/renku/service/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service.""" diff --git a/renku/service/config.py b/renku/service/config.py new file mode 100644 index 0000000000..a3c7393ea7 --- /dev/null +++ b/renku/service/config.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service config.""" +import os +import tempfile +from pathlib import Path + +INVALID_PARAMS_ERROR_CODE = -32602 +INTERNAL_FAILURE_ERROR_CODE = -32603 + +API_VERSION = 'v1' + +SWAGGER_URL = '/api/docs' +API_URL = os.getenv( + 'RENKU_SVC_SWAGGER_URL', '/api/{0}/spec'.format(API_VERSION) +) + +UPLOAD_FOLDER = tempfile.TemporaryDirectory() + +CACHE_UPLOADS_PATH = Path(UPLOAD_FOLDER.name) / Path('uploads') +CACHE_PROJECTS_PATH = Path(UPLOAD_FOLDER.name) / Path('projects') + +ALLOWED_EXTENSIONS = {'txt', 'pdf', 'csv'} + +JWT_ALGORITHM = 'HS256' +JWT_KEY = 'renku-svc-secret-key' diff --git a/renku/service/entrypoint.py b/renku/service/entrypoint.py new file mode 100644 index 0000000000..dc9895475a --- /dev/null +++ b/renku/service/entrypoint.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service entry point.""" +import os +import uuid + +from flask import Flask +from flask_swagger_ui import get_swaggerui_blueprint + +from renku.service.config import API_URL, API_VERSION, CACHE_PROJECTS_PATH, \ + CACHE_UPLOADS_PATH, SWAGGER_URL, UPLOAD_FOLDER +from renku.service.views.cache import clone_repository_view, \ + list_projects_view, list_uploaded_files_view, upload_files_view +from renku.service.views.datasets import add_file_to_dataset_view, \ + create_dataset_view, list_dataset_files_view, list_datasets_view +from renku.service.views.docs import api_docs_view + + +def make_cache(): + """Create cache structure.""" + sub_dirs = [CACHE_UPLOADS_PATH, CACHE_PROJECTS_PATH] + + for subdir in sub_dirs: + if not subdir.exists(): + subdir.mkdir() + + +def create_app(): + """Creates a Flask app with necessary configuration.""" + app = Flask(__name__) + app.secret_key = os.getenv('RENKU_SVC_SERVICE_KEY', uuid.uuid4().hex) + + app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER.name + app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 + + make_cache() + build_routes(app) + + return app + + +def build_routes(app): + """Register routes to given app instance.""" + swaggerui_blueprint = get_swaggerui_blueprint( + SWAGGER_URL, API_URL, config={'app_name': 'RenkuSvc'} + ) + + app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL) + + app.add_url_rule( + '/api/{0}/spec'.format(API_VERSION), + 'docs', + api_docs_view, + methods=['GET'] + ) + + app.add_url_rule( + '/cache/files-list', + 'list_files', + list_uploaded_files_view, + methods=['GET'] + ) + + app.add_url_rule( + '/cache/files-upload', + 'upload_files', + upload_files_view, + methods=['POST'] + ) + + app.add_url_rule( + '/cache/project-clone', + 'clone_project', + clone_repository_view, + methods=['GET'] + ) + + app.add_url_rule( + '/cache/project-list', + 'list_projects', + list_projects_view, + methods=['GET'] + ) + + app.add_url_rule( + '/datasets/list', 'list_datasets', list_datasets_view, methods=['GET'] + ) + + app.add_url_rule( + '/datasets/files', + 'list_dataset_files', + list_dataset_files_view, + methods=['GET'] + ) + + app.add_url_rule( + '/datasets/add', + 'add_dataset_file', + add_file_to_dataset_view, + methods=['GET'] + ) + + app.add_url_rule( + '/datasets/create', + 'create_data', + create_dataset_view, + methods=['GET'] + ) + + +app = create_app() + +if __name__ == '__main__': + app.run() diff --git a/renku/service/management/__init__.py b/renku/service/management/__init__.py new file mode 100644 index 0000000000..dfe37b4559 --- /dev/null +++ b/renku/service/management/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service management commands.""" diff --git a/renku/service/management/openapi.py b/renku/service/management/openapi.py new file mode 100644 index 0000000000..3b89ea0707 --- /dev/null +++ b/renku/service/management/openapi.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate OpenAPI spec for Renku service.""" +import json +from pathlib import Path + +from apispec import APISpec +from apispec_webframeworks.flask import FlaskPlugin + +import renku.service.views.cache as cache_views +import renku.service.views.datasets as datasets_views +from renku.service.entrypoint import create_app + + +def generate_openapi(): + """Generate OpenAPI spec.""" + app = create_app() + + spec = APISpec( + title='Renku as a Service', + openapi_version='3.0.2', + version='0.7.1', + info=dict(description='You know, for devs'), + plugins=[FlaskPlugin()] + ) + + with app.test_request_context(): + spec.path(view=cache_views.clone_repository_view) + spec.path(view=cache_views.list_projects_view) + spec.path(view=cache_views.upload_files_view) + spec.path(view=cache_views.list_uploaded_files_view) + + spec.path(view=datasets_views.list_dataset_files_view) + spec.path(view=datasets_views.list_datasets_view) + spec.path(view=datasets_views.add_file_to_dataset_view) + spec.path(view=datasets_views.create_dataset_view) + + spec_file = Path(__file__).absolute().parent / '../static/spec.json' + with open(spec_file, 'w') as f: + json.dump(spec.to_dict(), f) + + +if __name__ == '__main__': + generate_openapi() diff --git a/renku/service/static/spec.json b/renku/service/static/spec.json new file mode 100644 index 0000000000..0486f3dec9 --- /dev/null +++ b/renku/service/static/spec.json @@ -0,0 +1,339 @@ +{ + "info": { + "description": "You know, for devs", + "title": "Renku as a Service", + "version": "0.7.1" + }, + "paths": { + "/cache/project-clone": { + "get": { + "tags": [ + "cache" + ], + "summary": "Clone remote repository.", + "description": "Clone remote repository specified by git_url argument.", + "parameters": [ + { + "in": "header", + "name": "Authorization", + "schema": { + "type": "string" + }, + "required": true, + "description": "User identification." + }, + { + "in": "query", + "name": "git_url", + "description": "Remote URL of git repository.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "in": "query", + "name": "git_username", + "description": "Username to access remote git repository.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "in": "query", + "name": "git_access_token", + "description": "Access token for remote git repository.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "in": "query", + "name": "force", + "description": "Delete currently cached project and reclone it.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "result": { + "description": "Contains `project_id` reference." + }, + "error": { + "description": "Contains `code` and `message` describing the error." + } + } + } + }, + "/cache/project-list": { + "get": { + "tags": [ + "cache" + ], + "summary": "List cached repositories.", + "description": "List cached repositories.", + "parameters": [ + { + "in": "header", + "name": "Authorization", + "schema": { + "type": "string" + }, + "required": true, + "description": "User identification." + } + ], + "responses": { + "result": { + "description": "Contains `projects` key with list of all cached projects.\n" + }, + "error": { + "description": "Contains `code` and `message` describing the error." + } + } + } + }, + "/cache/files-upload": { + "post": { + "tags": [ + "cache" + ], + "summary": "Upload a file.", + "description": null, + "parameters": [ + { + "in": "header", + "name": "Authorization", + "schema": { + "type": "string" + }, + "required": true, + "description": "User identification." + }, + { + "in": "header", + "name": "file", + "description": "File parts (file uploaded via multipart).", + "schema": { + "type": "array" + }, + "required": true + }, + { + "in": "query", + "name": "unpack_archive", + "description": "Determines if renku should work with archive.", + "schema": { + "type": "bool" + }, + "required": false + } + ], + "responses": { + "result": { + "description": "Contains `file_id` reference." + }, + "error": { + "description": "Contains `code` and `message` \\ describing the error." + } + } + } + }, + "/cache/files-list": { + "get": { + "tags": [ + "cache" + ], + "summary": "List of uploaded files.", + "description": "List of cached files.", + "parameters": [ + { + "in": "header", + "name": "Authorization", + "schema": { + "type": "string" + }, + "required": true, + "description": "User identification." + } + ], + "responses": { + "result": { + "description": "Contains `files` key which contains\nlist of all files for a given user.\n" + }, + "error": { + "description": "Contains `code` and `message` describing the error." + } + } + } + }, + "/datasets/files": { + "get": { + "tags": [ + "datasets" + ], + "summary": "List all dataset files within a project.", + "description": "List all dataset files within a project.", + "parameters": [ + { + "name": "project_id", + "in": "query", + "description": "Project identifier.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "dataset_id", + "in": "query", + "description": "Dataset identifier.", + "schema": { + "type": "string" + }, + "required": true + } + ], + "responses": { + "result": { + "description": "Contains `datasets` key with\nlist of all datasets from a specified project.\n" + }, + "error": { + "description": "Contains `code` and `message`\ndescribing the error." + } + } + } + }, + "/datasets/list": { + "get": { + "tags": [ + "datasets" + ], + "summary": "List all datasets within a project.", + "description": "List all datasets within a project.", + "parameters": [ + { + "name": "project_id", + "in": "query", + "description": "Project identifier.", + "schema": { + "type": "string" + }, + "required": true + } + ], + "responses": { + "result": { + "description": "Contains `datasets` key with\nlist of all datasets from a specified project.\n" + }, + "error": { + "description": "Contains `code` and `message`\ndescribing the error." + } + } + } + }, + "/datasets/add": { + "get": { + "tags": [ + "datasets" + ], + "summary": "Add a file from cache to cloned repository.", + "description": "Add a file from cache to cloned repository", + "parameters": [ + { + "name": "dataset_name", + "in": "query", + "description": "Name of the dataset.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "file_id", + "in": "query", + "description": "File identifier to add to project.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "project_id", + "in": "query", + "description": "Project identifier.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "remote_name", + "in": "query", + "description": "Name of the remote to sync the project with.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "result": { + "description": "Contains `dataset` and `file_id` keys used in operation.\n" + }, + "error": { + "description": "Contains `code` and `message` describing the error." + } + } + } + }, + "/datasets/create": { + "get": { + "tags": [ + "datasets" + ], + "summary": "Create a new dataset with cached projects.", + "description": "Create a new dataset with cached projects.", + "parameters": [ + { + "name": "project_id", + "in": "query", + "description": "Project identifier.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "dataset_name", + "in": "query", + "description": "Dataset name.", + "schema": { + "type": "string" + }, + "required": true + }, + { + "name": "remote_name", + "in": "query", + "description": "Name of the remote to sync the project with.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "result": { + "description": "Contains `dataset` name of created dataset." + }, + "error": { + "description": "Contains `code` and `message` describing the error." + } + } + } + } + }, + "openapi": "3.0.2" + } \ No newline at end of file diff --git a/renku/service/views/__init__.py b/renku/service/views/__init__.py new file mode 100644 index 0000000000..dfb9f90cb2 --- /dev/null +++ b/renku/service/views/__init__.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service views.""" +import os +import re +from functools import wraps + +import jwt +from flask import request, jsonify +from git import Repo + +from renku.service.config import ALLOWED_EXTENSIONS, CACHE_PROJECTS_PATH, \ + INVALID_PARAMS_ERROR_CODE, CACHE_UPLOADS_PATH, JWT_KEY, JWT_ALGORITHM + +is_git_url = re.compile( + r'((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?', + flags=re.I +) + + +def allowed_file(filename): + """Check if filename is allowed.""" + has_extension = '.' in filename + is_allowed = filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + return has_extension and is_allowed + + +def find_file(user, file_id): + """Search for cached file.""" + for filename in os.listdir(CACHE_UPLOADS_PATH): + metadata = jwt.decode(filename, JWT_KEY, algorithms=[JWT_ALGORITHM]) + if metadata['user'] == user and file_id == filename.split('.')[1]: + return CACHE_UPLOADS_PATH / filename + + +def validate_git_url(git_url): + """Parse Git URL.""" + if not git_url or not is_git_url.match(git_url): + return None + + name = git_url.split('/')[-1].split('.')[0] + return name + + +def validate_project_id(user, project_id): + """Validate if given project id is in cache.""" + if project_id and (CACHE_PROJECTS_PATH / user / project_id).exists(): + return CACHE_PROJECTS_PATH / user / project_id + + +def repo_sync(repo_path, remote_name): + """Sync the repo with the default remote.""" + repo = Repo(repo_path) + is_pushed = False + + for remote in repo.remotes: + if remote.name == remote_name: + remote.push() + is_pushed = True + + return is_pushed + + +def requires_identity(f): + """Wrapper which indicates that route requires user identification.""" + # noqa + @wraps(f) + def decorated_function(*args, **kws): + """Represents decorated function.""" + user = request.headers.get('Authorization', '').split(' ')[-1] + if not user: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'user authorization is missing' + } + ) + + return f(user, *args, **kws) + + return decorated_function diff --git a/renku/service/views/cache.py b/renku/service/views/cache.py new file mode 100644 index 0000000000..c427c002e8 --- /dev/null +++ b/renku/service/views/cache.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache views.""" +import os +import shutil +import time +from pathlib import Path +from urllib.parse import urlparse + +import jwt +from flask import jsonify, request +from git import Repo +from werkzeug.utils import secure_filename + +from renku.service.config import CACHE_PROJECTS_PATH, CACHE_UPLOADS_PATH, \ + INTERNAL_FAILURE_ERROR_CODE, INVALID_PARAMS_ERROR_CODE, JWT_ALGORITHM, \ + JWT_KEY +from renku.service.views import allowed_file, requires_identity, \ + validate_git_url + + +@requires_identity +def list_uploaded_files_view(user): + """ + List uploaded files ready to be added to projects. + + --- + get: + tags: + - "cache" + + summary: List of uploaded files. + description: List of cached files. + + parameters: + - in: header + name: Authorization + schema: + type: string + required: true + description: User identification. + + responses: + result: + description: | + Contains `files` key which contains + list of all files for a given user. + + error: + description: | + Contains `code` and `message` describing the error. + + """ + if request.method == 'GET': + files = [] + for file_name in os.listdir(CACHE_UPLOADS_PATH): + file_path = CACHE_UPLOADS_PATH / file_name + size = os.stat(file_path).st_size + + metadata = jwt.decode( + file_name, JWT_KEY, algorithms=[JWT_ALGORITHM] + ) + + if metadata['user'] == user: + id_ = file_name.split('.')[1] + files.append({id_: {'size': size}}) + + return jsonify(result={'files': files}) + + +@requires_identity +def upload_files_view(user): + r""" + Handle file upload. + + --- + post: + tags: + - "cache" + + summary: Upload a file. + description: + + parameters: + - in: header + name: Authorization + schema: + type: string + required: true + description: User identification. + + - in: header + name: file + description: File parts (file uploaded via multipart). + schema: + type: array + required: true + + - in: query + name: unpack_archive + description: Determines if renku should work with archive. + schema: + type: bool + required: false + + responses: + result: + description: Contains `file_id` reference. + + error: + description: Contains `code` and `message` \ + describing the error. + + """ + if request.method == 'POST': + + if 'file' not in request.files: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'no file parts' + } + ) + + file = request.files['file'] + if file.filename == '': + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'no file selected for uploading' + } + ) + + if file and allowed_file(file.filename): + unpack_archive = request.args.get('unpack_archive', False) + filename = secure_filename(file.filename) + timestamp = time.time() * 1e+9 + + file_metadata = { + 'filename': filename, + 'timestamp': timestamp, + 'user': user, + 'unpack_archive': unpack_archive + } + + encoded = jwt.encode( + file_metadata, JWT_KEY, algorithm=JWT_ALGORITHM + ) + + local_path = CACHE_UPLOADS_PATH / Path(encoded.decode('utf-8')) + file.save(str(local_path)) + file_id = encoded.decode('utf-8').split('.')[1] + + return jsonify(result={'file_id': file_id}) + + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'file type not allowed' + } + ) + + +@requires_identity +def clone_repository_view(user): + """ + Clone remote repository. + + --- + get: + tags: + - "cache" + + summary: Clone remote repository. + description: Clone remote repository specified by git_url argument. + + parameters: + - in: header + name: Authorization + schema: + type: string + required: true + description: User identification. + + - in: query + name: git_url + description: Remote URL of git repository. + schema: + type: string + required: true + + - in: query + name: git_username + description: Username to access remote git repository. + schema: + type: string + required: true + + - in: query + name: git_access_token + description: Access token for remote git repository. + schema: + type: string + required: true + + - in: query + name: force + description: Delete currently cached project and reclone it. + schema: + type: boolean + + responses: + result: + description: Contains `project_id` reference. + + error: + description: | + Contains `code` and `message` describing the error. + + """ + if request.method == 'GET': + git_url = request.args.get('git_url') + git_username = request.args.get('git_username') + git_access_token = request.args.get('git_access_token') + + force = 'force' in request.args + + name = validate_git_url(git_url) + git_url = urlparse(git_url) + url_with_auth = '{0}:{1}@{2}'.format( + git_username, git_access_token, git_url.netloc + ) + git_url = git_url._replace(netloc=url_with_auth).geturl() + + if not name: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid git url' + } + ) + + local_path = CACHE_PROJECTS_PATH / user / name + if local_path.exists() and not force: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': ( + 'repository already cloned, ' + 'use force parameter to override cache state' + ) + } + ) + + if local_path.exists() and force: + shutil.rmtree(local_path) + + Repo.clone_from(git_url, str(local_path)) + + if not local_path.exists(): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'cloning failed', + } + ) + + return jsonify(result={ + 'project_id': name, + }) + + +@requires_identity +def list_projects_view(user): + """ + List cached projects. + + --- + get: + tags: + - "cache" + + summary: List cached repositories. + description: List cached repositories. + + parameters: + - in: header + name: Authorization + schema: + type: string + required: true + description: User identification. + + responses: + result: + description: | + Contains `projects` key with list of all cached projects. + + error: + description: | + Contains `code` and `message` describing the error. + + """ + if request.method == 'GET': + user = request.headers.get('Authorization', '').split(' ')[-1] + if not user: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'user authorization is missing' + } + ) + + projects = [] + project_dir = CACHE_PROJECTS_PATH / user + if project_dir.exists(): + for project in os.listdir(project_dir): + projects.append(project) + + return jsonify(result={'projects': projects}) diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py new file mode 100644 index 0000000000..b25a5b5dbb --- /dev/null +++ b/renku/service/views/datasets.py @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service datasets view.""" +import json +import os + +from flask import jsonify, request + +from renku.core.commands.dataset import add_file, create_dataset, \ + dataset_parent, list_files +from renku.service.config import INTERNAL_FAILURE_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE +from renku.service.views import find_file, repo_sync, requires_identity, \ + validate_project_id + + +@requires_identity +def list_datasets_view(user): + """ + List all datasets in specified project id. + + --- + get: + tags: + - "datasets" + + summary: List all datasets within a project. + description: List all datasets within a project. + + parameters: + - name: project_id + in: query + description: Project identifier. + schema: + type: string + required: true + + responses: + result: + description: | + Contains `datasets` key with + list of all datasets from a specified project. + + error: + description: | + Contains `code` and `message` + describing the error. + + """ + if request.method == 'GET': + project_id = request.args.get('project_id') + pwd = validate_project_id(project_id) + if not pwd: + return jsonify(status=2, message='invalid project_id argument') + + cwd = os.getcwd() + os.chdir(str(pwd)) + + datasets = [{ + 'id': ds['_label'], + 'name': ds['name'], + 'version': ds['version'], + 'tags': ds['tags'] + } for ds in json.loads(dataset_parent(None, 'data', 'json-ld'))] + + os.chdir(cwd) + return jsonify(datasets=datasets) + + +@requires_identity +def list_dataset_files_view(user): + """ + List all dataset files in specified project id. + + --- + get: + tags: + - "datasets" + + summary: List all dataset files within a project. + description: List all dataset files within a project. + + parameters: + - name: project_id + in: query + description: Project identifier. + schema: + type: string + required: true + + - name: dataset_id + in: query + description: Dataset identifier. + schema: + type: string + required: true + + responses: + result: + description: | + Contains `datasets` key with + list of all datasets from a specified project. + + error: + description: | + Contains `code` and `message` + describing the error. + + """ + if request.method == 'GET': + dataset_name = request.args.get('dataset_id') + project_id = request.args.get('project_id') + pwd = validate_project_id(project_id) + if not pwd: + return jsonify(status=2, message='invalid project_id argument') + + cwd = os.getcwd() + os.chdir(str(pwd)) + + files = [ + ds for ds in + json.loads(list_files(dataset_name, None, None, None, 'json-ld')) + ] + + os.chdir(cwd) + return jsonify(dataset=dataset_name, files=files) + + +@requires_identity +def add_file_to_dataset_view(user): + """ + Add uploaded file to cloned repository. + + --- + get: + tags: + - "datasets" + + summary: Add a file from cache to cloned repository. + description: Add a file from cache to cloned repository + + parameters: + - name: dataset_name + in: query + description: Name of the dataset. + schema: + type: string + required: true + + - name: file_id + in: query + description: File identifier to add to project. + schema: + type: string + required: true + + - name: project_id + in: query + description: Project identifier. + schema: + type: string + required: true + + - name: remote_name + in: query + description: Name of the remote to sync the project with. + schema: + type: string + + responses: + result: + description: | + Contains `dataset` and `file_id` keys used in operation. + + error: + description: | + Contains `code` and `message` describing the error. + + """ + if request.method == 'GET': + dataset_name = request.args.get('dataset_name') + project_id = request.args.get('project_id') + file_id = request.args.get('file_id') + remote_name = request.args.get('remote_name', 'origin') + + local_path = find_file(user, file_id) + if not local_path or not local_path.exists(): + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid file_id argument' + } + ) + + pwd = validate_project_id(user, project_id) + if not pwd: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id argument', + } + ) + + cwd = os.getcwd() + os.chdir(str(pwd)) + try: + add_file([str(local_path)], + dataset_name, + use_external_storage=False) + if not repo_sync(pwd, remote_name): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'remote name not found' + } + ) + + except (Exception, BaseException) as e: + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'failed to add file or sync with remote', + 'exception': str(e) + } + ) + + finally: + os.chdir(cwd) + + return jsonify(result={ + 'dataset': dataset_name, + 'file_id': file_id, + }) + + +@requires_identity +def create_dataset_view(user): + """ + Create a new dataset in cloned repository. + + --- + get: + tags: + - "datasets" + + summary: Create a new dataset with cached projects. + description: Create a new dataset with cached projects. + + parameters: + - name: project_id + in: query + description: Project identifier. + schema: + type: string + required: true + + - name: dataset_name + in: query + description: Dataset name. + schema: + type: string + required: true + + - name: remote_name + in: query + description: Name of the remote to sync the project with. + schema: + type: string + + responses: + result: + description: Contains `dataset` name of created dataset. + + error: + description: | + Contains `code` and `message` describing the error. + + """ + if request.method == 'GET': + dataset_name = request.args.get('dataset_name') + project_id = request.args.get('project_id') + pwd = validate_project_id(user, project_id) + + remote_name = request.args.get('remote_name', 'origin') + + if not pwd: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id argument', + } + ) + + cwd = os.getcwd() + os.chdir(str(pwd)) + try: + create_dataset(dataset_name) + if not repo_sync(pwd, remote_name): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'remote name not found' + } + ) + finally: + os.chdir(cwd) + + return jsonify(result={ + 'dataset': dataset_name, + }) diff --git a/renku/service/views/docs.py b/renku/service/views/docs.py new file mode 100644 index 0000000000..5c0d0cedf2 --- /dev/null +++ b/renku/service/views/docs.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service docs view.""" +import json +from pathlib import Path + +from flask import jsonify + + +def api_docs_view(): + """Serve API docs.""" + spec_file = Path(__file__).absolute().parent / '../static/spec.json' + payload = json.loads(spec_file.read_text()) + return jsonify(payload) diff --git a/setup.py b/setup.py index 304b36d25a..40358b1102 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,8 @@ install_requires = [ 'appdirs>=1.4.3', + 'apispec==3.0.0', + 'apispec-webframeworks==0.5.0', 'attrs>=18.2.0', 'click-completion>=0.5.0', 'click>=7.0', @@ -80,11 +82,14 @@ 'cwltool==1.0.20181012180214', 'environ_config>=18.2.0', 'filelock>=3.0.0', + 'flask==1.1.1', + 'flask-swagger-ui==3.20.9', 'gitpython==3.0.3', 'patool>=1.12', 'psutil>=5.4.7', 'pyasn1>=0.4.5', 'PyYAML>=3.12', + 'pyjwt==1.7.1', 'pyld>=1.0.3', 'pyOpenSSL>=19.0.0', 'python-dateutil>=2.6.1', diff --git a/tests/service/test_views.py b/tests/service/test_views.py new file mode 100644 index 0000000000..9de0aa1064 --- /dev/null +++ b/tests/service/test_views.py @@ -0,0 +1,316 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service view tests.""" +import io + +import pytest + +from renku.service.config import INVALID_PARAMS_ERROR_CODE + + +@pytest.mark.service +def test_serve_api_spec(svc_client): + """Check serving of service spec.""" + response = svc_client.get('/api/v1/spec') + + assert 0 != len(response.json.keys()) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all(svc_client): + """Check list uploaded files view.""" + headers_user = {'Authorization': 'bearer user'} + response = svc_client.get('/cache/files-list', headers=headers_user) + + assert 'error' not in response.json + assert 'result' in response.json + + assert 0 == len(response.json['result']['files']) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all_no_auth(svc_client): + """Check error response on list uploaded files view.""" + response = svc_client.get('/cache/files-list') + + assert 200 == response.status_code + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'result' not in response.json + + +@pytest.mark.service +def test_file_upload(svc_client): + """Check successful file upload.""" + headers_user = {'Authorization': 'bearer user'} + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + assert 'error' not in response.json + assert 'result' in response.json + assert response.json['result']['file_id'] + + +@pytest.mark.service +def test_file_upload_no_auth(svc_client): + """Check failed file upload.""" + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + ) + + assert response + assert 200 == response.status_code + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'result' not in response.json + + +@pytest.mark.service +def test_file_upload_with_users(svc_client): + """Check successful file upload and listing based on user auth header.""" + headers_user1 = {'Authorization': 'bearer user1'} + headers_user2 = {'Authorization': 'bearer user2'} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers_user1 + ) + assert 'error' not in response.json + + file_id = response.json['result']['file_id'] + assert file_id + assert 200 == response.status_code + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile2.txt'), ), + headers=headers_user2 + ) + + assert response + assert 'error' not in response.json + + response = svc_client.get('/cache/files-list', headers=headers_user1) + + assert response + assert 'error' not in response.json + + assert 1 == len(response.json['result']['files']) + + file = response.json['result']['files'][0] + + for name, metadata in file.items(): + assert name == file_id + assert 0 != metadata['size'] + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_no_auth(svc_client): + """Check error on cloning of remote repository.""" + access_token = 'contact:EcfPJvEqjJepyu6XyqKZ' + remote_url = 'https://{0}@renkulab.io/gitlab/contact/integration-tests.git' + + params = {'git_url': remote_url.format(access_token)} + response = svc_client.get('/cache/project-clone', query_string=params) + + assert response + assert 'error' in response.json + assert 'result' not in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_with_auth(svc_client): + """Check cloning of remote repository.""" + remote_url = 'https://renkulab.io/gitlab/contact/integration-tests.git' + headers = {'Authorization': 'bearer b4b4de0eda0f471ab82702bd5c367fa7'} + + params = { + 'git_username': 'contact', + 'git_access_token': 'EcfPJvEqjJepyu6XyqKZ', + 'git_url': remote_url + } + + response = svc_client.get( + '/cache/project-clone', query_string=params, headers=headers + ) + + assert response + assert 'error' not in response.json + assert 'result' in response.json + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_view_errors(svc_client): + """Check cache state of cloned projects/""" + access_token = 'contact:EcfPJvEqjJepyu6XyqKZ' + remote_url = 'https://{0}@renkulab.io/gitlab/contact/integration-tests.git' + headers = {'Authorization': 'bearer b4b4de0eda0f471ab82702bd5c367fa7'} + + params = {'git_url': remote_url.format(access_token)} + + response = svc_client.get( + '/cache/project-clone', query_string=params, headers=headers + ) + assert response + + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'use force' in response.json['error']['message'] + + params['force'] = 1 + response = svc_client.get( + '/cache/project-clone', query_string=params, headers=headers + ) + assert response + assert 'error' not in response.json + assert 'result' in response.json + assert 'integration-tests' == response.json['result']['project_id'] + + response = svc_client.get( + '/cache/project-list', + # no auth headers, expected error + ) + assert response + + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'result' not in response.json + + response = svc_client.get('/cache/project-list', headers=headers) + assert response + + assert 'error' not in response.json + assert 'result' in response.json + assert ['integration-tests'] == response.json['result']['projects'] + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view(svc_client_with_repo): + """Create new dataset.""" + svc_client, headers = svc_client_with_repo + + params = { + 'project_id': 'integration-tests', + 'dataset_name': 'my-dataset', + 'remote_name': 'origin', + } + + response = svc_client.get( + '/datasets/create', + query_string=params, + headers=headers, + ) + + assert response + assert 'result' in response.json + assert 'error' not in response.json + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view_with_err(svc_client_with_repo): + """Create new dataset view invoke user error.""" + svc_client, _ = svc_client_with_repo + + params = { + 'project_id': 'integration-tests', + 'dataset_name': 'my-dataset', + 'remote_name': 'origin', + } + + response = svc_client.get( + '/datasets/create', + query_string=params, + # no user identity, expect error + ) + + assert response + assert 'result' not in response.json + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view_with_err(svc_client_with_repo): + """Check error raise in dataset add.""" + svc_client, headers = svc_client_with_repo + params = { + 'project_id': 'integration-tests', + 'dataset_name': 'my-dataset', + 'remote_name': 'origin', + } + + response = svc_client.get( + '/datasets/add', + query_string=params, + # no user identity, expect error + ) + + assert response + assert 'result' not in response.json + assert 'error' in response.json + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view(svc_client_with_repo): + """Check adding of uploaded file to dataset.""" + svc_client, headers = svc_client_with_repo + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers + ) + assert 'error' not in response.json + + file_id = response.json['result']['file_id'] + assert file_id + assert 200 == response.status_code + + params = { + 'project_id': 'integration-tests', + 'dataset_name': 'my-dataset', + 'remote_name': 'origin', + 'file_id': file_id, + } + response = svc_client.get( + '/datasets/add', + query_string=params, + headers=headers, + ) + + assert response + assert 'result' in response.json + assert 'error' not in response.json