diff --git a/.env b/.env new file mode 100644 index 0000000000..222989c353 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DATABASE=0 +REDIS_PASSWORD= diff --git a/.gitignore b/.gitignore index 9c0ccc6751..56245fde97 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ target/ renku-*.bottle.json renku-*.bottle.tar.gz renku.rb + +.env diff --git a/.travis.yml b/.travis.yml index db5a1653d8..70eaaa17be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ before_install: - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get update; sudo apt-get -y install shellcheck; - travis_retry python -m pip install --upgrade pip setuptools py; + travis_retry python -m pip install --upgrade six pip setuptools py; travis_retry python -m pip install twine wheel coveralls requirements-builder; requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt; requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt; diff --git a/Dockerfile b/Dockerfile.cli similarity index 96% rename from Dockerfile rename to Dockerfile.cli index 770a6e39ee..6e63b77185 100644 --- a/Dockerfile +++ b/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM python:3.6-alpine as base +FROM python:3.7-alpine as base RUN apk add --no-cache git && \ pip install --no-cache --upgrade pip diff --git a/Dockerfile.svc b/Dockerfile.svc new file mode 100644 index 0000000000..dc7283df4d --- /dev/null +++ b/Dockerfile.svc @@ -0,0 +1,18 @@ +FROM python:3.7-alpine + +RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git bash && \ + pip install --no-cache --upgrade pip setuptools pipenv requirements-builder + +RUN apk add --no-cache --allow-untrusted \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \ + --repository http://nl.alpinelinux.org/alpine/edge/community \ + git-lfs && \ + git lfs install + +COPY . /code/renku +WORKDIR /code/renku +RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn + + +ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"] diff --git a/MANIFEST.in b/MANIFEST.in index 8c9e058fa7..d77b546611 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,15 +17,17 @@ # limitations under the License. # Check manifest will not automatically add these two files: +include renku/service/.env-example include .dockerignore include .editorconfig include .tx/config include *.md prune docs/_build recursive-include renku *.po *.pot *.mo - +recursive-include renku *.py # added by check_manifest.py include *.py +include *.yml include *.rst include *.sh include *.txt @@ -39,6 +41,7 @@ include babel.ini include brew.py include pytest.ini include snap/snapcraft.yaml +recursive-include renku *.json recursive-include .github CODEOWNERS recursive-include .travis *.sh recursive-include docs *.bat @@ -60,3 +63,4 @@ recursive-include renku *.json recursive-include renku Dockerfile recursive-include tests *.py *.gz *.yml *.json prune .github +prune .env diff --git a/Makefile b/Makefile index 2602ce90e6..9e04a816c8 100644 --- a/Makefile +++ b/Makefile @@ -69,3 +69,6 @@ brew-commit-bottle: *.bottle.json brew-release: open "https://github.com/SwissDataScienceCenter/renku-python/releases/new?tag=v$(shell brew info --json=v1 renku | jq -r '.[0].versions.stable')" + +service-container: + docker build -f Dockerfile.svc -t renku-svc:`git rev-parse --short HEAD` . diff --git a/Pipfile.lock b/Pipfile.lock index 936bc038a6..559749827d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -30,6 +30,13 @@ ], "version": "==1.5" }, + "apispec": { + "hashes": [ + "sha256:5fdaa1173b32515cc83f9d413a49a6c37fafc2b87f6b40e95923d3e85f0942c5", + "sha256:9e88c51517a6515612e818459f61c1bc06c00f2313e5187828bdbabaa7461473" + ], + "version": "==3.0.0" + }, "appdirs": { "hashes": [ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", @@ -37,26 +44,12 @@ ], "version": "==1.4.3" }, - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "avro-cwl": { "hashes": [ @@ -87,43 +80,48 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" }, "chardet": { "hashes": [ @@ -134,10 +132,10 @@ }, "check-manifest": { "hashes": [ - "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", - "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" + "sha256:42de6eaab4ed149e60c9b367ada54f01a3b1e4d6846784f9b9710e770ff5572c", + "sha256:78dd077f2c70dbac7cfcc9d12cbd423914e787ea4b5631de45aecd25b524e8e3" ], - "version": "==0.39" + "version": "==0.40" }, "click": { "hashes": [ @@ -148,9 +146,9 @@ }, "click-completion": { "hashes": [ - "sha256:78072eecd5e25ea0d25ceaf99cd5f22aa2667d67231ae0819deab9b1ff3456fb" + "sha256:5bf816b81367e638a190b6e91b50779007d14301b3f9f3145d68e3cade7bce86" ], - "version": "==0.5.1" + "version": "==0.5.2" }, "coverage": { "hashes": [ @@ -191,24 +189,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" - ], - "version": "==2.7" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" }, "cwlref-runner": { "hashes": [ @@ -226,10 +229,10 @@ }, "decorator": { "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" ], - "version": "==4.4.0" + "version": "==4.4.1" }, "docutils": { "hashes": [ @@ -260,6 +263,13 @@ ], "version": "==1.7.1" }, + "fakeredis": { + "hashes": [ + "sha256:169598943dc10aadd62871a34b2867bb5e24f9da7ebc97a2058c3f35c760241e", + "sha256:1db27ec3a5c964b9fb9f36ec1b9770a81204c54e84f83c763f36689eef4a5fd4" + ], + "version": "==1.1.0" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -269,10 +279,30 @@ }, "flake8": { "hashes": [ - "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", - "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "version": "==3.7.9" + }, + "flask": { + "hashes": [ + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + ], + "version": "==1.1.1" + }, + "flask-apispec": { + "hashes": [ + "sha256:46bb89f8c4be3547d3f48536100f88a2a249ae59b050589cff57a0ec8e25d000", + "sha256:b97a9d7200293021ff11fa393157f51736dc12d6b4fc4502140561fb3cf64a16" ], - "version": "==3.7.8" + "version": "==0.8.3" + }, + "flask-swagger-ui": { + "hashes": [ + "sha256:3282c770764c8053360f33b2fc120e1d169ecca2138537d0e6e1135b1f9d4ff2" + ], + "version": "==3.20.9" }, "freezegun": { "hashes": [ @@ -319,11 +349,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", + "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.1.0" }, "isodate": { "hashes": [ @@ -340,6 +370,13 @@ ], "version": "==4.3.4" }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, "jinja2": { "hashes": [ "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", @@ -356,30 +393,34 @@ }, "lxml": { "hashes": [ - "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4", - "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc", - "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1", - "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046", - "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36", - "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5", - "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d", - "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916", - "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0", - "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27", - "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc", - "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7", - "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38", - "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5", - "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832", - "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a", - "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f", - "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9", - "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692", - "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84", - "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79", - "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681" - ], - "version": "==4.4.1" + "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", + "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", + "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", + "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", + "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", + "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", + "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", + "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", + "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", + "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", + "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", + "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", + "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", + "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", + "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", + "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", + "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", + "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", + "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", + "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", + "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", + "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", + "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", + "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", + "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", + "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" + ], + "version": "==4.4.2" }, "markupsafe": { "hashes": [ @@ -414,6 +455,14 @@ ], "version": "==1.1.1" }, + "marshmallow": { + "hashes": [ + "sha256:1a358beb89c2b4d5555272065a9533591a3eb02f1b854f3c4002d88d8f2a1ddb", + "sha256:eb97c42c5928b5720812c9268865fe863d4807bc1a8b48ddd7d5c9e1779a6af0" + ], + "markers": "python_version >= '3'", + "version": "==3.2.2" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -430,16 +479,17 @@ }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" ], - "version": "==7.2.0" + "version": "==8.0.0" }, "mypy-extensions": { "hashes": [ - "sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458" + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" ], - "version": "==0.4.2" + "version": "==0.4.3" }, "ndg-httpsclient": { "hashes": [ @@ -451,9 +501,17 @@ }, "networkx": { "hashes": [ - "sha256:8311ddef63cf5c5c5e7c1d0212dd141d9a1fe3f474915281b73597ed5f1d4e3d" + "sha256:cdfbf698749a5014bf2ed9db4a07a5295df1d3a53bf80bf3cbd61edf9df05fa1", + "sha256:f8f4ff0b6f96e4f9b16af6b84622597b5334bf9cae8cf9b2e42e7985d5c95c64" ], - "version": "==2.3" + "version": "==2.4" + }, + "owlrl": { + "hashes": [ + "sha256:2ad753f5ba6d1fe2d88bf36f427df31553f2eaa0283692e3cd06cab20ac8aec3", + "sha256:efdebe76cf9ad148f316a9ae92e898e12b3b3690bd90218c898a2b676955b266" + ], + "version": "==5.2.1" }, "packaging": { "hashes": [ @@ -478,10 +536,10 @@ }, "pluggy": { "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.13.0" + "version": "==0.13.1" }, "prov": { "hashes": [ @@ -492,17 +550,19 @@ }, "psutil": { "hashes": [ - "sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000", - "sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d", - "sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3", - "sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7", - "sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45", - "sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61", - "sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76", - "sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63", - "sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a" + "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b", + "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806", + "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b", + "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995", + "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd", + "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73", + "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465", + "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d", + "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a", + "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217", + "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa" ], - "version": "==5.6.3" + "version": "==5.6.7" }, "py": { "hashes": [ @@ -513,10 +573,10 @@ }, "pyasn1": { "hashes": [ - "sha256:62cdade8b5530f0b185e09855dd422bc05c0bbff6b72ff61381c09dac7befd8c", - "sha256:a9495356ca1d66ed197a0f72b41eb1823cf7ea8b5bd07191673e8147aecf8604" + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" ], - "version": "==0.4.7" + "version": "==0.4.8" }, "pycodestyle": { "hashes": [ @@ -547,10 +607,10 @@ }, "pygments": { "hashes": [ - "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", - "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.4.2" + "version": "==2.5.2" }, "pyld": { "hashes": [ @@ -560,24 +620,31 @@ }, "pyopenssl": { "hashes": [ - "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200", - "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6" + "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", + "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" ], - "version": "==19.0.0" + "version": "==19.1.0" }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" ], - "version": "==2.4.2" + "version": "==2.4.5" + }, + "pyshacl": { + "hashes": [ + "sha256:74739aa88dcdb161849f769d28e3a14ed0b01eb8256a21cc2941a27dc80e70af", + "sha256:d551c0e5842f30151b3e398407f5ffc8250e602add925f0bb31bce5771419169" + ], + "version": "==0.11.3.post1" }, "pytest": { "hashes": [ - "sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8", - "sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0" + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" ], - "version": "==5.2.1" + "version": "==5.3.1" }, "pytest-cache": { "hashes": [ @@ -607,10 +674,10 @@ }, "pytest-runner": { "hashes": [ - "sha256:25a013c8d84f0ca60bb01bd11913a3bcab420f601f0f236de4423074af656e7a", - "sha256:d04243fbf29a3b574f18f1bcff2a07f505db5daede82f706f2e32728f77d3f4d" + "sha256:5534b08b133ef9a5e2c22c7886a8f8508c95bb0b0bdc6cc13214f269c3c70d51", + "sha256:96c7e73ead7b93e388c5d614770d2bae6526efd997757d3543fe17b557a0942b" ], - "version": "==5.1" + "version": "==5.2" }, "pytest-yapf": { "hashes": [ @@ -622,10 +689,10 @@ }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "python-editor": { "hashes": [ @@ -673,6 +740,13 @@ ], "version": "==0.4.0" }, + "redis": { + "hashes": [ + "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", + "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" + ], + "version": "==3.3.11" + }, "renku": { "editable": true, "extras": [ @@ -696,10 +770,10 @@ }, "responses": { "hashes": [ - "sha256:502d9c0c8008439cfcdef7e251f507fcfdd503b56e8c0c87c3c3e3393953f790", - "sha256:97193c0183d63fba8cd3a041c75464e4b09ea0aff6328800d1546598567dde0b" + "sha256:46d4e546a19fc6106bc7e804edd4551ef04690405e41e7e750ebc295d042623b", + "sha256:93b1e0f2f960c0f3304ca4436856241d64c33683ef441431b9caf1d05d9d9e23" ], - "version": "==0.10.6" + "version": "==0.10.7" }, "ruamel.yaml": { "hashes": [ @@ -753,10 +827,10 @@ }, "sentry-sdk": { "hashes": [ - "sha256:15e51e74b924180c98bcd636cb4634945b0a99a124d50b433c3a9dc6a582e8db", - "sha256:1d6a2ee908ec6d8f96c27d78bc39e203df4d586d287c233140af7d8d1aca108a" + "sha256:a7c2c8d3f53b6b57454830cd6a4b73d272f1ba91952f59e6545b3cf885f3c22f", + "sha256:bfc486af718c268cf49ff43d6334ed4db7333ace420240b630acdd8f8a3a8f60" ], - "version": "==0.12.3" + "version": "==0.13.4" }, "setuptools-scm": { "hashes": [ @@ -781,10 +855,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "smmap2": { "hashes": [ @@ -800,12 +874,19 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "sphinx": { "hashes": [ - "sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845", - "sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069" + "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", + "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" ], - "version": "==2.2.0" + "version": "==2.2.1" }, "sphinxcontrib-applehelp": { "hashes": [ @@ -859,9 +940,9 @@ }, "tabulate": { "hashes": [ - "sha256:d0097023658d4dea848d6ae73af84532d1e86617ac0925d1adf1dd903985dac3" + "sha256:5470cc6687a091c7042cee89b2946d9235fe9f6d49c193a4ae2ac7bf386737c8" ], - "version": "==0.8.5" + "version": "==0.8.6" }, "toml": { "hashes": [ @@ -872,18 +953,18 @@ }, "tqdm": { "hashes": [ - "sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d", - "sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115" + "sha256:156a0565f09d1f0ef8242932a0e1302462c93827a87ba7b4423d90f01befe94c", + "sha256:c0ffb55959ea5f3eaeece8d2db0651ba9ced9c72f40a6cce3419330256234289" ], - "version": "==4.36.1" + "version": "==4.40.0" }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], - "version": "==3.7.4" + "version": "==3.7.4.1" }, "unify": { "hashes": [ @@ -899,10 +980,10 @@ }, "urllib3": { "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.6" + "version": "==1.25.7" }, "wcwidth": { "hashes": [ @@ -911,6 +992,14 @@ ], "version": "==0.1.7" }, + "webargs": { + "hashes": [ + "sha256:3beca296598067cec24a0b6f91c0afcc19b6e3c4d84ab026b931669628bb47b4", + "sha256:3f9dc15de183d356c9a0acc159c100ea0506c0c240c1e6f1d8b308c5fed4dbbd", + "sha256:fa4ad3ad9b38bedd26c619264fdc50d7ae014b49186736bca851e5b5228f2a1b" + ], + "version": "==5.5.2" + }, "werkzeug": { "hashes": [ "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", diff --git a/conftest.py b/conftest.py index a0e4fa49bd..63f880b524 100644 --- a/conftest.py +++ b/conftest.py @@ -25,8 +25,10 @@ import tempfile import time import urllib +import uuid from pathlib import Path +import fakeredis import pytest import responses import yaml @@ -510,3 +512,176 @@ def remote_project(data_repository, directory_tree): assert 0 == result.exit_code yield runner, project_path + + +@pytest.fixture(scope='function') +def dummy_datapack(): + """Creates dummy data folder.""" + temp_dir = tempfile.TemporaryDirectory() + + data_file_txt = Path(temp_dir.name) / Path('file.txt') + data_file_txt.write_text('my awesome data') + + data_file_csv = Path(temp_dir.name) / Path('file.csv') + data_file_csv.write_text('more,awesome,data') + + yield temp_dir + + +@pytest.fixture(scope='function') +def datapack_zip(dummy_datapack): + """Returns dummy data folder as a zip archive.""" + from renku.core.utils.contexts import chdir + workspace_dir = tempfile.TemporaryDirectory() + with chdir(workspace_dir.name): + shutil.make_archive('datapack', 'zip', dummy_datapack.name) + + yield Path(workspace_dir.name) / 'datapack.zip' + + +@pytest.fixture(scope='function') +def datapack_tar(dummy_datapack): + """Returns dummy data folder as a tar archive.""" + from renku.core.utils.contexts import chdir + workspace_dir = tempfile.TemporaryDirectory() + with chdir(workspace_dir.name): + shutil.make_archive('datapack', 'tar', dummy_datapack.name) + + yield Path(workspace_dir.name) / 'datapack.tar' + + +@pytest.fixture(scope='function') +def mock_redis(monkeypatch): + """Monkey patch service cache with mocked redis.""" + from renku.service.cache import ServiceCache + with monkeypatch.context() as m: + m.setattr(ServiceCache, 'cache', fakeredis.FakeRedis()) + yield + + +@pytest.fixture(scope='function') +def svc_client(mock_redis): + """Renku service client.""" + from renku.service.entrypoint import create_app + + flask_app = create_app() + + testing_client = flask_app.test_client() + testing_client.testing = True + + ctx = flask_app.app_context() + ctx.push() + + yield testing_client + + ctx.pop() + + +@pytest.fixture(scope='function') +def svc_client_with_repo(svc_client, mock_redis): + """Renku service remote repository.""" + remote_url = 'https://dev.renku.ch/gitlab/contact/integration-tests' + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': 'b4b4de0eda0f471ab82702bd5c367fa7', + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer LkoLiyLqnhMCAa4or5qa', + } + + payload = {'git_url': remote_url} + + response = svc_client.post( + '/cache/project-clone', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert 'result' in response.json + assert 'error' not in response.json + project_id = response.json['result']['project_id'] + assert isinstance(uuid.UUID(project_id), uuid.UUID) + + yield svc_client, headers, project_id + + +@pytest.fixture( + params=[ + { + 'url': '/cache/files-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/cache/files-upload', + 'allowed_method': 'POST', + 'headers': {} + }, + { + 'url': '/cache/project-clone', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/cache/project-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/add', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/create', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/files-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + ] +) +def service_allowed_endpoint(request, svc_client, mock_redis): + """Ensure allowed methods and correct headers.""" + methods = { + 'GET': svc_client.get, + 'POST': svc_client.post, + 'HEAD': svc_client.head, + 'PUT': svc_client.put, + 'DELETE': svc_client.delete, + 'OPTIONS': svc_client.options, + 'TRACE': svc_client.trace, + 'PATCH': svc_client.patch, + } + + yield methods, request.param, svc_client diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..96436d1b24 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + redis: + image: redis:5.0.3-alpine + ports: + - "6379:6379" + + renku-svc: + image: renku-svc:latest + env_file: .env + ports: + - "8080:8080" diff --git a/renku/cli/__init__.py b/renku/cli/__init__.py index e6dfe7d750..1021fcb38d 100644 --- a/renku/cli/__init__.py +++ b/renku/cli/__init__.py @@ -91,7 +91,7 @@ option_use_external_storage from renku.core.commands.version import check_version, print_version from renku.core.management.client import LocalClient -from renku.core.management.config import ConfigManagerMixin, RENKU_HOME +from renku.core.management.config import RENKU_HOME, ConfigManagerMixin from renku.core.management.repository import default_path #: Monkeypatch Click application. diff --git a/renku/core/commands/client.py b/renku/core/commands/client.py index 12d103fae0..f9eec0d0e2 100644 --- a/renku/core/commands/client.py +++ b/renku/core/commands/client.py @@ -25,6 +25,8 @@ import yaml from renku.core.management import LocalClient +from renku.core.management.config import RENKU_HOME +from renku.core.management.repository import default_path from .git import get_git_isolation @@ -63,8 +65,17 @@ def pass_local_client( ) def new_func(*args, **kwargs): - ctx = click.get_current_context() - client = ctx.ensure_object(LocalClient) + ctx = click.get_current_context(silent=True) + if not ctx: + client = LocalClient( + path=default_path(), + renku_home=RENKU_HOME, + use_external_storage=True, + ) + ctx = click.Context(click.Command(method)) + else: + client = ctx.ensure_object(LocalClient) + stack = contextlib.ExitStack() # Handle --isolation option: @@ -85,8 +96,11 @@ def new_func(*args, **kwargs): if lock or (lock is None and commit): stack.enter_context(client.lock) - with stack: - result = ctx.invoke(method, client, *args, **kwargs) + result = None + if ctx: + with stack: + result = ctx.invoke(method, client, *args, **kwargs) + return result return functools.update_wrapper(new_func, method) diff --git a/renku/core/commands/clone.py b/renku/core/commands/clone.py index d3805640a5..003f2a792a 100644 --- a/renku/core/commands/clone.py +++ b/renku/core/commands/clone.py @@ -29,7 +29,11 @@ def renku_clone( path=None, install_githooks=True, skip_smudge=True, - progress=None + recursive=True, + depth=None, + progress=None, + config=None, + raise_git_except=False ): """Clone Renku project repo, install Git hooks and LFS.""" install_lfs = client.use_external_storage @@ -39,5 +43,9 @@ def renku_clone( install_githooks=install_githooks, install_lfs=install_lfs, skip_smudge=skip_smudge, - progress=progress + recursive=recursive, + depth=depth, + progress=progress, + config=config, + raise_git_except=raise_git_except, ) diff --git a/renku/core/commands/dataset.py b/renku/core/commands/dataset.py index 6604b2838a..ef7734c2ec 100644 --- a/renku/core/commands/dataset.py +++ b/renku/core/commands/dataset.py @@ -146,7 +146,7 @@ def add_file( destination='', ref=None, with_metadata=None, - urlscontext=contextlib.nullcontext + urlscontext=contextlib.nullcontext, ): """Add data file to a dataset.""" add_to_dataset( diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py index 5c4f900587..4de5f8f86c 100644 --- a/renku/core/management/clone.py +++ b/renku/core/management/clone.py @@ -18,6 +18,7 @@ """Clone a Renku repo along with all Renku-specific initializations.""" import os +from pathlib import Path from git import GitCommandError, Repo @@ -33,23 +34,46 @@ def clone( skip_smudge=True, recursive=True, depth=None, - progress=None + progress=None, + config=None, + raise_git_except=False, ): """Clone Renku project repo, install Git hooks and LFS.""" from renku.core.management.client import LocalClient path = path or '.' + + if isinstance(path, Path): + path = str(path) + # Clone the project if skip_smudge: os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' + try: repo = Repo.clone_from( url, path, recursive=recursive, depth=depth, progress=progress ) except GitCommandError as e: - raise errors.GitError( - 'Cannot clone remote Renku project: {}'.format(url) - ) from e + if not raise_git_except: + raise errors.GitError( + 'Cannot clone remote Renku project: {}'.format(url) + ) from e + + raise e + + if config: + config_writer = repo.config_writer() + + for key, value in config.items(): + key_path = key.split('.') + if len(key_path) != 2: + raise errors.GitError( + 'Cannot write to config path: {0}'.format(key) + ) + config_writer.set_value(key_path[0], key_path[1], value) + + config_writer.release() client = LocalClient(path) diff --git a/renku/core/management/repository.py b/renku/core/management/repository.py index a937390738..114081ed40 100644 --- a/renku/core/management/repository.py +++ b/renku/core/management/repository.py @@ -47,13 +47,18 @@ def default_path(): return '.' +def path_converter(path): + """Converter for path in PathMixin.""" + return Path(path).resolve() + + @attr.s class PathMixin: """Define a default path attribute.""" path = attr.ib( default=default_path, - converter=lambda arg: Path(arg).resolve().absolute(), + converter=path_converter, ) @path.validator diff --git a/renku/core/utils/contexts.py b/renku/core/utils/contexts.py index 77de0bc61b..06131ade45 100644 --- a/renku/core/utils/contexts.py +++ b/renku/core/utils/contexts.py @@ -26,6 +26,9 @@ @contextlib.contextmanager def chdir(path): """Change the current working directory.""" + if isinstance(path, Path): + path = str(path) + cwd = os.getcwd() os.chdir(path) try: diff --git a/renku/service/.env-example b/renku/service/.env-example new file mode 100644 index 0000000000..45f635b6c3 --- /dev/null +++ b/renku/service/.env-example @@ -0,0 +1,7 @@ +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DATABASE=0 +REDIS_PASSWORD= + +CACHE_DIR= +PROJECT_CLONE_DEPTH_DEFAULT=1 diff --git a/renku/service/__init__.py b/renku/service/__init__.py new file mode 100644 index 0000000000..1928b35350 --- /dev/null +++ b/renku/service/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service.""" diff --git a/renku/service/cache/__init__.py b/renku/service/cache/__init__.py new file mode 100644 index 0000000000..ca18ae78ec --- /dev/null +++ b/renku/service/cache/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache management for files.""" +from renku.service.cache.files import FileManagementCache +from renku.service.cache.projects import ProjectManagementCache + + +class ServiceCache(FileManagementCache, ProjectManagementCache): + """Service cache manager.""" + + pass diff --git a/renku/service/cache/base.py b/renku/service/cache/base.py new file mode 100644 index 0000000000..c38f3f8d79 --- /dev/null +++ b/renku/service/cache/base.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache management.""" +import json + +import redis +from redis import RedisError + +from renku.service.cache.config import REDIS_DATABASE, REDIS_HOST, \ + REDIS_PASSWORD, REDIS_PORT + + +class BaseCache: + """Cache management.""" + + cache = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DATABASE, + password=REDIS_PASSWORD + ) + + def set_record(self, name, key, value): + """Insert a record to hash set.""" + if isinstance(value, dict): + value = json.dumps(value) + + self.cache.hset(name, key, value) + + def invalidate_key(self, name, key): + """Invalidate cache `key` in users hash set.""" + try: + self.cache.hdel(name, key) + except RedisError: + pass + + def get_record(self, name, key): + """Return record values from hash set.""" + result = self.cache.hget(name, key) + if result: + return json.loads(result.decode('utf-8')) + + def get_all_records(self, name): + """Return all record values from hash set.""" + return [ + json.loads(record.decode('utf-8')) + for record in self.cache.hgetall(name).values() + ] diff --git a/renku/service/cache/config.py b/renku/service/cache/config.py new file mode 100644 index 0000000000..7afb2d6b68 --- /dev/null +++ b/renku/service/cache/config.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache configuration.""" +import os + +REDIS_HOST = os.getenv('REDIS_HOST', '0.0.0.0') +REDIS_PORT = int(os.getenv('REDIS_PORT', 6379)) +REDIS_DATABASE = int(os.getenv('REDIS_DATABASE', 0)) +REDIS_PASSWORD = os.getenv('REDIS_PASSWORD') diff --git a/renku/service/cache/files.py b/renku/service/cache/files.py new file mode 100644 index 0000000000..ab8240c79e --- /dev/null +++ b/renku/service/cache/files.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service files cache management.""" +from renku.service.cache.base import BaseCache + + +class FileManagementCache(BaseCache): + """File management cache.""" + + FILES_SUFFIX = 'files' + + def files_cache_key(self, user): + """Construct cache key based on user and files suffix.""" + return '{0}_{1}'.format(user, self.FILES_SUFFIX) + + def set_file(self, user, file_id, metadata): + """Cache file metadata under user hash set.""" + self.set_record(self.files_cache_key(user), file_id, metadata) + + def set_files(self, user, files): + """Cache a list of metadata files under user hash set.""" + for file_ in files: + self.set_file(user, file_['file_id'], file_) + + def get_files(self, user): + """Get all user cached files.""" + return self.get_all_records(self.files_cache_key(user)) + + def get_file(self, user, file_id): + """Get user cached file.""" + result = self.get_record(self.files_cache_key(user), file_id) + return result + + def invalidate_file(self, user, file_id): + """Remove file record from hash set.""" + self.invalidate_key(self.files_cache_key(user), file_id) diff --git a/renku/service/cache/projects.py b/renku/service/cache/projects.py new file mode 100644 index 0000000000..21f10e9a8a --- /dev/null +++ b/renku/service/cache/projects.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service project cache management.""" +from renku.service.cache.base import BaseCache + + +class ProjectManagementCache(BaseCache): + """Project management cache.""" + + PROJECTS_SUFFIX = 'projects' + + def projects_cache_key(self, user): + """Construct cache key based on user and projects suffix.""" + return '{0}_{1}'.format(user, self.PROJECTS_SUFFIX) + + def set_project(self, user, project_id, metadata): + """Cache project metadata under user hash set.""" + self.set_record(self.projects_cache_key(user), project_id, metadata) + + def get_projects(self, user): + """Get all user cache projects.""" + return self.get_all_records(self.projects_cache_key(user)) + + def get_project(self, user, project_id): + """Get user cached project.""" + result = self.get_record(self.projects_cache_key(user), project_id) + return result + + def invalidate_project(self, user, project_id): + """Remove project record from hash set.""" + self.invalidate_key(self.projects_cache_key(user), project_id) diff --git a/renku/service/config.py b/renku/service/config.py new file mode 100644 index 0000000000..774f2c775a --- /dev/null +++ b/renku/service/config.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service config.""" +import os +import tempfile +from pathlib import Path + +GIT_ACCESS_DENIED_ERROR_CODE = -32000 +GIT_UNKNOWN_ERROR_CODE = -32001 + +RENKU_EXCEPTION_ERROR_CODE = -32100 +REDIS_EXCEPTION_ERROR_CODE = -32200 + +INVALID_HEADERS_ERROR_CODE = -32601 +INVALID_PARAMS_ERROR_CODE = -32602 +INTERNAL_FAILURE_ERROR_CODE = -32603 + +SERVICE_NAME = 'Renku Service' +OPENAPI_VERSION = '2.0' +API_VERSION = 'v1' + +SWAGGER_URL = '/api/docs' +API_SPEC_URL = os.getenv( + 'RENKU_SVC_SWAGGER_URL', '/api/{0}/spec'.format(API_VERSION) +) + +PROJECT_CLONE_DEPTH_DEFAULT = int(os.getenv('PROJECT_CLONE_DEPTH_DEFAULT', 1)) + +CACHE_DIR = os.getenv('CACHE_DIR', tempfile.TemporaryDirectory().name) +CACHE_UPLOADS_PATH = Path(CACHE_DIR) / Path('uploads') +CACHE_UPLOADS_PATH.mkdir(parents=True, exist_ok=True) + +CACHE_PROJECTS_PATH = Path(CACHE_DIR) / Path('projects') +CACHE_PROJECTS_PATH.mkdir(parents=True, exist_ok=True) + +TAR_ARCHIVE_CONTENT_TYPE = 'application/x-tar' +ZIP_ARCHIVE_CONTENT_TYPE = 'application/zip' + +SUPPORTED_ARCHIVES = [ + TAR_ARCHIVE_CONTENT_TYPE, + ZIP_ARCHIVE_CONTENT_TYPE, +] diff --git a/renku/service/entrypoint.py b/renku/service/entrypoint.py new file mode 100644 index 0000000000..7f645c9e90 --- /dev/null +++ b/renku/service/entrypoint.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service entry point.""" +import os +import uuid + +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin +from flask import Flask +from flask_apispec import FlaskApiSpec +from flask_swagger_ui import get_swaggerui_blueprint + +from renku.service.cache import ServiceCache +from renku.service.config import API_SPEC_URL, API_VERSION, CACHE_DIR, \ + CACHE_PROJECTS_PATH, CACHE_UPLOADS_PATH, OPENAPI_VERSION, SERVICE_NAME, \ + SWAGGER_URL +from renku.service.views.cache import CACHE_BLUEPRINT_TAG, cache_blueprint, \ + list_projects_view, list_uploaded_files_view, project_clone, \ + upload_file_view +from renku.service.views.datasets import DATASET_BLUEPRINT_TAG, \ + add_file_to_dataset_view, create_dataset_view, dataset_blueprint, \ + list_dataset_files_view, list_datasets_view + + +def make_cache(): + """Create cache structure.""" + sub_dirs = [CACHE_UPLOADS_PATH, CACHE_PROJECTS_PATH] + + for subdir in sub_dirs: + if not subdir.exists(): + subdir.mkdir() + + return ServiceCache() + + +def create_app(): + """Creates a Flask app with necessary configuration.""" + app = Flask(__name__) + app.secret_key = os.getenv('RENKU_SVC_SERVICE_KEY', uuid.uuid4().hex) + + app.config['UPLOAD_FOLDER'] = CACHE_DIR + app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 + + cache = make_cache() + app.config['cache'] = cache + + build_routes(app) + return app + + +def build_routes(app): + """Register routes to given app instance.""" + app.config.update({ + 'APISPEC_SPEC': + APISpec( + title=SERVICE_NAME, + openapi_version=OPENAPI_VERSION, + version=API_VERSION, + plugins=[MarshmallowPlugin()], + ), + 'APISPEC_SWAGGER_URL': API_SPEC_URL, + }) + app.register_blueprint(cache_blueprint) + app.register_blueprint(dataset_blueprint) + + swaggerui_blueprint = get_swaggerui_blueprint( + SWAGGER_URL, API_SPEC_URL, config={'app_name': 'Renku Service'} + ) + app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL) + + docs = FlaskApiSpec(app) + + docs.register(upload_file_view, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(list_uploaded_files_view, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(project_clone, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(list_projects_view, blueprint=CACHE_BLUEPRINT_TAG) + + docs.register(create_dataset_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(add_file_to_dataset_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(list_datasets_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(list_dataset_files_view, blueprint=DATASET_BLUEPRINT_TAG) + + +app = create_app() + +if __name__ == '__main__': + app.run() diff --git a/renku/service/serializers/__init__.py b/renku/service/serializers/__init__.py new file mode 100644 index 0000000000..362f6221d1 --- /dev/null +++ b/renku/service/serializers/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service serializers.""" diff --git a/renku/service/serializers/cache.py b/renku/service/serializers/cache.py new file mode 100644 index 0000000000..c42415245e --- /dev/null +++ b/renku/service/serializers/cache.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache serializers.""" +import time +import uuid +from urllib.parse import urlparse + +from marshmallow import Schema, ValidationError, fields, post_load, pre_load, \ + validates +from werkzeug.utils import secure_filename + +from renku.core.errors import ConfigurationError +from renku.core.models.git import GitURL +from renku.service.config import PROJECT_CLONE_DEPTH_DEFAULT +from renku.service.serializers.rpc import JsonRPCResponse + + +def extract_file(request): + """Extract file from Flask request. + + :raises: `ValidationError` + """ + files = request.files + if 'file' not in files: + raise ValidationError('missing key: file') + + file = files['file'] + if file and not file.filename: + raise ValidationError('wrong filename: {0}'.format(file.filename)) + + if file: + file.filename = secure_filename(file.filename) + return file + + +class FileUploadRequest(Schema): + """Request schema for file upload.""" + + override_existing = fields.Boolean(missing=False) + unpack_archive = fields.Boolean(missing=False) + + +class FileUploadContext(Schema): + """Context schema for file upload.""" + + file_id = fields.String(missing=lambda: uuid.uuid4().hex) + + # measured in ms + timestamp = fields.Integer(missing=time.time() * 1e+3) + + content_type = fields.String(missing='unknown') + file_name = fields.String(required=True) + + # measured in bytes (comes from stat() - st_size) + file_size = fields.Integer(required=True) + + relative_path = fields.String(required=True) + is_archive = fields.Boolean(missing=False) + unpack_archive = fields.Boolean(missing=False) + + +class FileUploadResponse(Schema): + """Response schema for file upload.""" + + files = fields.List(fields.Nested(FileUploadContext), required=True) + + +class FileUploadResponseRPC(JsonRPCResponse): + """RPC response schema for file upload response.""" + + result = fields.Nested(FileUploadResponse) + + +class FileListResponse(Schema): + """Response schema for files listing.""" + + files = fields.List(fields.Nested(FileUploadContext), required=True) + + +class FileListResponseRPC(JsonRPCResponse): + """RPC response schema for files listing.""" + + result = fields.Nested(FileListResponse) + + +class ProjectCloneRequest(Schema): + """Request schema for project clone.""" + + git_url = fields.String(required=True) + depth = fields.Integer(missing=PROJECT_CLONE_DEPTH_DEFAULT) + + +class ProjectCloneContext(ProjectCloneRequest): + """Context schema for project clone.""" + + project_id = fields.String(missing=lambda: uuid.uuid4().hex) + name = fields.String(required=True) + fullname = fields.String(required=True) + email = fields.String(required=True) + owner = fields.String(required=True) + token = fields.String(required=True) + + @validates('git_url') + def validate_git_url(self, value): + """Validates git url.""" + try: + GitURL.parse(value) + except ConfigurationError as e: + raise ValidationError(str(e)) + + return value + + @post_load() + def format_url(self, data, **kwargs): + """Format URL with username and password.""" + git_url = urlparse(data['git_url']) + + url = 'oauth2:{0}@{1}'.format(data['token'], git_url.netloc) + data['url_with_auth'] = git_url._replace(netloc=url).geturl() + + return data + + @pre_load() + def set_owner_name(self, data, **kwargs): + """Set owner and name fields.""" + git_url = GitURL.parse(data['git_url']) + + data['owner'] = git_url.owner + data['name'] = git_url.name + + return data + + +class ProjectCloneResponse(Schema): + """Response schema for project clone.""" + + project_id = fields.String(required=True) + git_url = fields.String(required=True) + + +class ProjectCloneResponseRPC(JsonRPCResponse): + """RPC response schema for project clone response.""" + + result = fields.Nested(ProjectCloneResponse) + + +class ProjectListResponse(Schema): + """Response schema for project listing.""" + + projects = fields.List(fields.Nested(ProjectCloneResponse), required=True) + + +class ProjectListResponseRPC(JsonRPCResponse): + """RPC response schema for project listing.""" + + result = fields.Nested(ProjectListResponse) diff --git a/renku/service/serializers/datasets.py b/renku/service/serializers/datasets.py new file mode 100644 index 0000000000..154a0a8623 --- /dev/null +++ b/renku/service/serializers/datasets.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service datasets serializers.""" +from marshmallow import Schema, fields + +from renku.service.serializers.rpc import JsonRPCResponse + + +class DatasetAuthors(Schema): + """Schema for the dataset authors.""" + + name = fields.String(required=True) + affiliation = fields.String() + + +class DatasetCreateRequest(Schema): + """Request schema for dataset create view.""" + + dataset_name = fields.String(required=True) + description = fields.String() + authors = fields.List(fields.Nested(DatasetAuthors)) + project_id = fields.String(required=True) + + +class DatasetCreateResponse(Schema): + """Response schema for dataset create view.""" + + dataset_name = fields.String(required=True) + + +class DatasetCreateResponseRPC(JsonRPCResponse): + """RPC response schema for dataset create view.""" + + result = fields.Nested(DatasetCreateResponse) + + +class DatasetAddFile(Schema): + """Schema for dataset add file view.""" + + file_id = fields.String(required=True) + + +class DatasetAddRequest(Schema): + """Request schema for dataset add file view.""" + + dataset_name = fields.String(required=True) + create_dataset = fields.Boolean(missing=False) + project_id = fields.String(required=True) + files = fields.List(fields.Nested(DatasetAddFile), required=True) + + +class DatasetAddResponse(Schema): + """Response schema for dataset add file view.""" + + dataset_name = fields.String(required=True) + project_id = fields.String(required=True) + files = fields.List(fields.Nested(DatasetAddFile), required=True) + + +class DatasetAddResponseRPC(JsonRPCResponse): + """RPC schema for dataset add.""" + + result = fields.Nested(DatasetAddResponse) + + +class DatasetListRequest(Schema): + """Request schema for dataset list view.""" + + project_id = fields.String(required=True) + + +class DatasetDetails(Schema): + """Serialize dataset to response object.""" + + identifier = fields.String(required=True) + name = fields.String(required=True) + version = fields.String(allow_none=True) + created = fields.String(allow_none=True) + + +class DatasetListResponse(Schema): + """Response schema for dataset list view.""" + + datasets = fields.List(fields.Nested(DatasetDetails), required=True) + + +class DatasetListResponseRPC(JsonRPCResponse): + """RPC response schema for dataset list view.""" + + result = fields.Nested(DatasetListResponse) + + +class DatasetFilesListRequest(Schema): + """Request schema for dataset files list view.""" + + project_id = fields.String(required=True) + dataset_name = fields.String(required=True) + + +class DatasetFileDetails(Schema): + """Serialzie dataset files to response object.""" + + name = fields.String(required=True) + + +class DatasetFilesListResponse(Schema): + """Response schema for dataset files list view.""" + + dataset_name = fields.String(required=True) + files = fields.List(fields.Nested(DatasetFileDetails), required=True) + + +class DatasetFilesListResponseRPC(JsonRPCResponse): + """RPC schema for dataset files list view.""" + + result = fields.Nested(DatasetFilesListResponse) diff --git a/renku/service/serializers/headers.py b/renku/service/serializers/headers.py new file mode 100644 index 0000000000..f1ab332fb0 --- /dev/null +++ b/renku/service/serializers/headers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service headers serializers.""" +from marshmallow import Schema, ValidationError, fields, pre_load +from werkzeug.utils import secure_filename + + +class UserIdentityHeaders(Schema): + """User identity schema.""" + + uid = fields.String(required=True, data_key='renku-user-id') + fullname = fields.String(data_key='renku-user-fullname') + email = fields.String(data_key='renku-user-email') + token = fields.String(data_key='authorization') + + def extract_token(self, data): + """Extract token.""" + value = data.get('authorization', '') + components = value.split(' ') + + rfc_compliant = value.lower().startswith('bearer') + rfc_compliant &= len(components) == 2 + + if not rfc_compliant: + raise ValidationError('authorization value contains invalid value') + + return components[-1] + + @pre_load() + def set_fields(self, data, **kwargs): + """Set fields for serialization.""" + expected_keys = [field.data_key for field in self.fields.values()] + + data = { + key.lower(): value + for key, value in data.items() if key.lower() in expected_keys + } + + if {'renku-user-id', 'authorization'}.issubset(set(data.keys())): + data['renku-user-id'] = secure_filename(data['renku-user-id']) + data['authorization'] = self.extract_token(data) + + return data diff --git a/renku/service/serializers/rpc.py b/renku/service/serializers/rpc.py new file mode 100644 index 0000000000..6512d26001 --- /dev/null +++ b/renku/service/serializers/rpc.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service JSON-RPC serializers.""" +from marshmallow import Schema, fields + + +class JsonRPCResponse(Schema): + """JsonRPC response schema.""" + + error = fields.Dict() diff --git a/renku/service/utils/__init__.py b/renku/service/utils/__init__.py new file mode 100644 index 0000000000..c5d87b539b --- /dev/null +++ b/renku/service/utils/__init__.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service utility functions.""" +from git import Repo + +from renku.service.config import CACHE_PROJECTS_PATH, CACHE_UPLOADS_PATH + + +def make_project_path(user, project): + """Construct full path for cached project.""" + valid_user = user and 'uid' in user + valid_project = project and 'owner' in project and 'name' in project + + if valid_user and valid_project: + return ( + CACHE_PROJECTS_PATH / user['uid'] / project['owner'] / + project['name'] + ) + + +def make_file_path(user, cached_file): + """Construct full path for cache file.""" + valid_user = user and 'uid' in user + valid_file = cached_file and 'file_name' in cached_file + + if valid_user and valid_file: + return CACHE_UPLOADS_PATH / user['uid'] / cached_file['relative_path'] + + +def repo_sync(repo_path, remote_names=('origin', )): + """Sync the repo with the remotes.""" + repo = Repo(repo_path) + is_pushed = False + + for remote in repo.remotes: + if remote.name in remote_names: + remote.push() + is_pushed = True + + return is_pushed diff --git a/renku/service/views/__init__.py b/renku/service/views/__init__.py new file mode 100644 index 0000000000..fbe49ab1d7 --- /dev/null +++ b/renku/service/views/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service views.""" diff --git a/renku/service/views/cache.py b/renku/service/views/cache.py new file mode 100644 index 0000000000..983c42d534 --- /dev/null +++ b/renku/service/views/cache.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache views.""" +import os +import shutil +from pathlib import Path + +import patoolib +from flask import Blueprint, jsonify, request +from flask_apispec import marshal_with, use_kwargs +from marshmallow import EXCLUDE +from patoolib.util import PatoolError + +from renku.core.commands.clone import renku_clone +from renku.service.config import CACHE_UPLOADS_PATH, \ + INVALID_PARAMS_ERROR_CODE, SUPPORTED_ARCHIVES +from renku.service.serializers.cache import FileListResponse, \ + FileListResponseRPC, FileUploadContext, FileUploadRequest, \ + FileUploadResponse, FileUploadResponseRPC, ProjectCloneContext, \ + ProjectCloneRequest, ProjectCloneResponse, ProjectCloneResponseRPC, \ + ProjectListResponse, ProjectListResponseRPC, extract_file +from renku.service.utils import make_file_path, make_project_path +from renku.service.views.decorators import accepts_json, handle_base_except, \ + handle_git_except, handle_renku_except, handle_validation_except, \ + header_doc, requires_cache, requires_identity + +CACHE_BLUEPRINT_TAG = 'cache' +cache_blueprint = Blueprint('cache', __name__) + + +@marshal_with(FileListResponseRPC) +@header_doc(description='List uploaded files.', tags=(CACHE_BLUEPRINT_TAG, )) +@cache_blueprint.route( + '/cache/files-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_uploaded_files_view(user, cache): + """List uploaded files ready to be added to projects.""" + files = [ + f for f in cache.get_files(user['uid']) + if make_file_path(user, f).exists() + ] + + response = FileListResponseRPC().load({ + 'result': FileListResponse().load({'files': files}) + }) + return jsonify(response) + + +@use_kwargs(FileUploadRequest) +@marshal_with(FileUploadResponseRPC) +@header_doc( + description='Upload file or archive of files.', + tags=(CACHE_BLUEPRINT_TAG, ), +) +@cache_blueprint.route( + '/cache/files-upload', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def upload_file_view(user, cache): + """Upload file or archive of files.""" + file = extract_file(request) + + response_builder = { + 'file_name': file.filename, + 'content_type': file.content_type, + 'is_archive': file.content_type in SUPPORTED_ARCHIVES + } + response_builder.update(FileUploadRequest().load(request.args)) + + user_cache_dir = CACHE_UPLOADS_PATH / user['uid'] + user_cache_dir.mkdir(exist_ok=True) + + file_path = user_cache_dir / file.filename + if file_path.exists(): + if response_builder.get('override_existing', False): + file_path.unlink() + else: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'file exists', + } + ) + + file.save(str(file_path)) + + files = [] + if response_builder['unpack_archive'] and response_builder['is_archive']: + unpack_dir = '{0}.unpacked'.format(file_path.name) + temp_dir = file_path.parent / Path(unpack_dir) + if temp_dir.exists(): + shutil.rmtree(str(temp_dir)) + temp_dir.mkdir(exist_ok=True) + + try: + patoolib.extract_archive(str(file_path), outdir=str(temp_dir)) + except PatoolError: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'unable to unpack archive' + } + ) + + for file_ in temp_dir.glob('**/*'): + file_obj = { + 'file_name': file_.name, + 'file_size': os.stat(str(file_path)).st_size, + 'relative_path': + str(file_.relative_to(CACHE_UPLOADS_PATH / user['uid'])) + } + + files.append(FileUploadContext().load(file_obj, unknown=EXCLUDE)) + + else: + response_builder['file_size'] = os.stat(str(file_path)).st_size + response_builder['relative_path'] = str( + file_path.relative_to(CACHE_UPLOADS_PATH / user['uid']) + ) + + files.append( + FileUploadContext().load(response_builder, unknown=EXCLUDE) + ) + + response = FileUploadResponseRPC().load({ + 'result': FileUploadResponse().load({'files': files}) + }) + cache.set_files(user['uid'], files) + + return jsonify(response) + + +@use_kwargs(ProjectCloneRequest) +@marshal_with(ProjectCloneResponseRPC) +@header_doc( + 'Clone a remote project. If the project is cached already, ' + 'new clone operation will override the old cache state.', + tags=(CACHE_BLUEPRINT_TAG, ) +) +@cache_blueprint.route( + '/cache/project-clone', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +@accepts_json +def project_clone(user, cache): + """Clone a remote repository.""" + ctx = ProjectCloneContext().load( + (lambda a, b: a.update(b) or a)(request.json, user), + unknown=EXCLUDE, + ) + + local_path = make_project_path(user, ctx) + + if local_path.exists(): + shutil.rmtree(str(local_path)) + + for project in cache.get_projects(user['uid']): + if project['git_url'] == ctx['git_url']: + cache.invalidate_project(user['uid'], project['project_id']) + + local_path.mkdir(parents=True, exist_ok=True) + renku_clone( + ctx['url_with_auth'], + local_path, + depth=ctx['depth'], + raise_git_except=True, + config={ + 'user.name': ctx['fullname'], + 'user.email': ctx['email'], + } + ) + cache.set_project(user['uid'], ctx['project_id'], ctx) + + response = ProjectCloneResponseRPC().load({ + 'result': ProjectCloneResponse().load(ctx, unknown=EXCLUDE) + }) + return jsonify(response) + + +@marshal_with(ProjectListResponseRPC) +@header_doc( + 'List cached projects.', + tags=(CACHE_BLUEPRINT_TAG, ), +) +@cache_blueprint.route( + '/cache/project-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_projects_view(user, cache): + """List cached projects.""" + projects = cache.get_projects(user['uid']) + projects = [ + ProjectCloneResponse().load(p, unknown=EXCLUDE) + for p in projects if make_project_path(user, p).exists() + ] + + response = ProjectListResponseRPC().load({ + 'result': ProjectListResponse().load({'projects': projects}) + }) + return jsonify(response) diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py new file mode 100644 index 0000000000..3984ee8877 --- /dev/null +++ b/renku/service/views/datasets.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service datasets view.""" +import json + +from flask import Blueprint, jsonify, request +from flask_apispec import marshal_with, use_kwargs +from marshmallow import EXCLUDE + +from renku.core.commands.dataset import add_file, create_dataset, \ + dataset_parent, list_files +from renku.core.utils.contexts import chdir +from renku.service.config import INTERNAL_FAILURE_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE +from renku.service.serializers.datasets import DatasetAddRequest, \ + DatasetAddResponse, DatasetAddResponseRPC, DatasetCreateRequest, \ + DatasetCreateResponse, DatasetCreateResponseRPC, DatasetDetails, \ + DatasetFileDetails, DatasetFilesListRequest, DatasetFilesListResponse, \ + DatasetFilesListResponseRPC, DatasetListRequest, DatasetListResponse, \ + DatasetListResponseRPC +from renku.service.utils import make_file_path, make_project_path, repo_sync +from renku.service.views.decorators import accepts_json, handle_base_except, \ + handle_git_except, handle_renku_except, handle_validation_except, \ + header_doc, requires_cache, requires_identity + +DATASET_BLUEPRINT_TAG = 'datasets' +dataset_blueprint = Blueprint(DATASET_BLUEPRINT_TAG, __name__) + + +@use_kwargs(DatasetListRequest, locations=['query']) +@marshal_with(DatasetListResponseRPC) +@header_doc('List all datasets in project.', tags=(DATASET_BLUEPRINT_TAG, )) +@dataset_blueprint.route( + '/datasets/list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_datasets_view(user, cache): + """List all datasets in project.""" + req = DatasetListRequest().load(request.args) + project = cache.get_project(user['uid'], req['project_id']) + project_path = make_project_path(user, project) + + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'invalid project_id argument', + } + ) + + with chdir(project_path): + datasets = [ + DatasetDetails().load(ds, unknown=EXCLUDE) + # TODO: fix core interface to address this issue (add ticket ref) + for ds in json.loads(dataset_parent(None, 'data', 'json-ld')) + ] + + response = DatasetListResponse().load({'datasets': datasets}) + return jsonify(DatasetListResponseRPC().load({'result': response})) + + +@use_kwargs(DatasetFilesListRequest, locations=['query']) +@marshal_with(DatasetFilesListResponseRPC) +@header_doc('List files in a dataset.', tags=(DATASET_BLUEPRINT_TAG, )) +@dataset_blueprint.route( + '/datasets/files-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_dataset_files_view(user, cache): + """List files in a dataset.""" + ctx = DatasetFilesListRequest().load(request.args) + project = cache.get_project(user['uid'], ctx['project_id']) + project_path = make_project_path(user, project) + + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'invalid project_id argument', + } + ) + + with chdir(project_path): + dataset_files = json.loads( + # TODO: fix core interface to address this issue (add ticket ref) + list_files(ctx['dataset_name'], None, None, None, 'json-ld') + ) + ctx['files'] = [ + DatasetFileDetails().load(ds, unknown=EXCLUDE) + for ds in dataset_files + ] + + response = DatasetFilesListResponse().load(ctx, unknown=EXCLUDE) + return jsonify(DatasetFilesListResponseRPC().load({'result': response})) + + +@use_kwargs(DatasetAddRequest) +@marshal_with(DatasetAddResponseRPC) +@header_doc( + 'Add uploaded file to cloned repository.', tags=(DATASET_BLUEPRINT_TAG, ) +) +@dataset_blueprint.route( + '/datasets/add', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@accepts_json +@requires_cache +@requires_identity +def add_file_to_dataset_view(user, cache): + """Add uploaded file to cloned repository.""" + ctx = DatasetAddRequest().load(request.json) + project = cache.get_project(user['uid'], ctx['project_id']) + project_path = make_project_path(user, project) + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id: {0}'.format(ctx['project_id']), + } + ) + + local_paths = [] + for file_ in ctx['files']: + file = cache.get_file(user['uid'], file_['file_id']) + local_path = make_file_path(user, file) + if not local_path or not local_path.exists(): + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid file_id: {0}'.format(file_['file_id']) + } + ) + local_paths.append(str(local_path)) + + with chdir(project_path): + add_file( + local_paths, ctx['dataset_name'], create=ctx['create_dataset'] + ) + + if not repo_sync(project_path): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'repo sync failed' + } + ) + + return jsonify( + DatasetAddResponseRPC().load({ + 'result': DatasetAddResponse().load(ctx, unknown=EXCLUDE) + }) + ) + + +@use_kwargs(DatasetCreateRequest) +@marshal_with(DatasetCreateResponseRPC) +@header_doc( + 'Create a new dataset in a project.', tags=(DATASET_BLUEPRINT_TAG, ) +) +@dataset_blueprint.route( + '/datasets/create', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@accepts_json +@requires_cache +@requires_identity +def create_dataset_view(user, cache): + """Create a new dataset in a project.""" + ctx = DatasetCreateRequest().load(request.json) + project = cache.get_project(user['uid'], ctx['project_id']) + + project_path = make_project_path(user, project) + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id argument', + } + ) + + with chdir(project_path): + create_dataset(ctx['dataset_name']) + + if not repo_sync(project_path): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'reason': 'push to remote failed silently - try again' + } + ) + + return jsonify( + DatasetCreateResponseRPC().load({ + 'result': DatasetCreateResponse().load(ctx, unknown=EXCLUDE) + }) + ) diff --git a/renku/service/views/decorators.py b/renku/service/views/decorators.py new file mode 100644 index 0000000000..517574bedc --- /dev/null +++ b/renku/service/views/decorators.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service view decorators.""" +from functools import wraps + +from flask import current_app, jsonify, request +from flask_apispec import doc +from git import GitCommandError +from marshmallow import ValidationError +from redis import RedisError + +from renku.core.errors import RenkuException +from renku.service.config import GIT_ACCESS_DENIED_ERROR_CODE, \ + GIT_UNKNOWN_ERROR_CODE, INTERNAL_FAILURE_ERROR_CODE, \ + INVALID_HEADERS_ERROR_CODE, INVALID_PARAMS_ERROR_CODE, \ + REDIS_EXCEPTION_ERROR_CODE, RENKU_EXCEPTION_ERROR_CODE +from renku.service.serializers.headers import UserIdentityHeaders + + +def requires_identity(f): + """Wrapper which indicates that route requires user identification.""" + # noqa + @wraps(f) + def decorated_function(*args, **kws): + """Represents decorated function.""" + try: + user = UserIdentityHeaders().load(request.headers) + except (ValidationError, KeyError): + err_message = 'user identification is incorrect or missing' + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': err_message + } + ) + + return f(user, *args, **kws) + + return decorated_function + + +def handle_redis_except(f): + """Wrapper which handles Redis exceptions.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except (RedisError, OSError) as e: + error_code = REDIS_EXCEPTION_ERROR_CODE + + return jsonify(error={ + 'code': error_code, + 'reason': e.messages, + }) + + return decorated_function + + +@handle_redis_except +def requires_cache(f): + """Wrapper which injects cache object into view.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + return f(current_app.config.get('cache'), *args, **kwargs) + + return decorated_function + + +def handle_validation_except(f): + """Wrapper which handles marshmallow `ValidationError`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except ValidationError as e: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': e.messages, + } + ) + + return decorated_function + + +def handle_renku_except(f): + """Wrapper which handles `RenkuException`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except RenkuException as e: + return jsonify( + error={ + 'code': RENKU_EXCEPTION_ERROR_CODE, + 'reason': str(e), + } + ) + + return decorated_function + + +def handle_git_except(f): + """Wrapper which handles `RenkuException`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except GitCommandError as e: + + error_code = GIT_ACCESS_DENIED_ERROR_CODE \ + if 'Access denied' in e.stderr else GIT_UNKNOWN_ERROR_CODE + + return jsonify( + error={ + 'code': error_code, + 'reason': + 'git error: {0}'. + format(' '.join(e.stderr.strip().split('\n'))), + } + ) + + return decorated_function + + +def accepts_json(f): + """Wrapper which ensures only JSON payload can be in request.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + if 'Content-Type' not in request.headers: + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': 'invalid request headers' + } + ) + + header_check = request.headers['Content-Type'] == 'application/json' + + if not request.is_json or not header_check: + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': 'invalid request payload' + } + ) + + return f(*args, **kwargs) + + return decorated_function + + +def handle_base_except(f): + """Wrapper which handles base exceptions.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except (Exception, BaseException, OSError) as e: + error_code = INTERNAL_FAILURE_ERROR_CODE + + return jsonify( + error={ + 'code': error_code, + 'reason': + 'internal error: {0}'. + format(' '.join(e.stderr.strip().split('\n'))), + } + ) + + return decorated_function + + +def header_doc(description, tags=()): + """Wrap additional OpenAPI header description for an endpoint.""" + return doc( + description=description, + params={ + 'Authorization': { + 'description': ( + 'Used for users git oauth2 access. ' + 'For example: ' + '```Authorization: Bearer asdf-qwer-zxcv```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-Id': { + 'description': ( + 'Used for identification of the users. ' + 'For example: ' + '```Renku-User-Id: sasdsa-sadsd-gsdsdh-gfdgdsd```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-FullName': { + 'description': ( + 'Used for commit author signature. ' + 'For example: ' + '```Renku-User-FullName: Rok Roskar```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-Email': { + 'description': ( + 'Used for commit author signature. ' + 'For example: ' + '```Renku-User-Email: dev@renkulab.io```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + }, + tags=list(tags), + ) diff --git a/setup.py b/setup.py index ae0d86b42e..8e799749d4 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,11 @@ tests_require = [ 'check-manifest>=0.37', 'coverage>=4.5.3', + 'fakeredis==1.1.0', 'flake8>=3.5', 'freezegun>=0.3.12', 'isort==4.3.4', + 'six>=1.13.0', 'pydocstyle>=3.0.0', 'pytest-cache>=1.0', 'pytest-cov>=2.5.1', @@ -73,6 +75,7 @@ install_requires = [ 'appdirs>=1.4.3', + 'apispec==3.0.0', 'attrs>=18.2.0', 'click-completion>=0.5.0', 'click>=7.0', @@ -80,6 +83,9 @@ 'cwltool==1.0.20181012180214', 'environ_config>=18.2.0', 'filelock>=3.0.0', + 'flask==1.1.1', + 'flask-apispec==0.8.3', + 'flask-swagger-ui==3.20.9', 'gitpython==3.0.3', 'patool>=1.12', 'psutil>=5.4.7', @@ -90,9 +96,11 @@ 'pyshacl>=0.11.3.post1', 'python-dateutil>=2.6.1', 'python-editor>=1.0.4', + 'redis==3.3.11', 'rdflib-jsonld>=0.4.0', 'requests>=2.21.0', 'ndg-httpsclient>=0.5.1', + 'marshmallow==3.2.2', 'idna>=2.8', 'setuptools_scm>=3.1.0', 'tabulate>=0.7.7', diff --git a/tests/cli/test_datasets.py b/tests/cli/test_datasets.py index a7efdb7890..23831ad075 100644 --- a/tests/cli/test_datasets.py +++ b/tests/cli/test_datasets.py @@ -1196,8 +1196,8 @@ def test_avoid_empty_commits(runner, client, directory_tree): def test_add_removes_credentials(runner, client): """Test credentials are removed when adding to a dataset.""" - URL = 'https://username:password@example.com/index.html' - result = runner.invoke(cli, ['dataset', 'add', '-c', 'my-dataset', URL]) + url = 'https://username:password@example.com/index.html' + result = runner.invoke(cli, ['dataset', 'add', '-c', 'my-dataset', url]) assert 0 == result.exit_code with client.with_dataset('my-dataset') as dataset: diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py new file mode 100644 index 0000000000..c4b23e0952 --- /dev/null +++ b/tests/service/test_cache_views.py @@ -0,0 +1,600 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache view tests.""" +import io +import json +import uuid + +import pytest + +from renku.core.models.git import GitURL +from renku.service.config import ( + INVALID_HEADERS_ERROR_CODE, INVALID_PARAMS_ERROR_CODE +) + +REMOTE_URL = 'https://dev.renku.ch/gitlab/contact/integration-tests' +PERSONAL_ACCESS_TOKEN = 'LkoLiyLqnhMCAa4or5qa' + + +@pytest.mark.service +def test_serve_api_spec(svc_client): + """Check serving of service spec.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + response = svc_client.get('/api/v1/spec', headers=headers) + + assert 0 != len(response.json.keys()) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all(svc_client): + """Check list uploaded files view.""" + headers_user = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': 'user' + } + response = svc_client.get('/cache/files-list', headers=headers_user) + + assert {'result'} == set(response.json.keys()) + + assert 0 == len(response.json['result']['files']) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all_no_auth(svc_client): + """Check error response on list uploaded files view.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + response = svc_client.get( + '/cache/files-list', + headers=headers, + ) + + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +def test_file_upload(svc_client): + """Check successful file upload.""" + headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + + +@pytest.mark.service +def test_file_upload_override(svc_client): + """Check successful file upload.""" + headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + old_file_id = response.json['result']['files'][0]['file_id'] + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'file exists' == response.json['error']['reason'] + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + query_string={'override_existing': True}, + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + assert old_file_id != response.json['result']['files'][0]['file_id'] + + +@pytest.mark.service +def test_file_upload_same_file(svc_client): + """Check successful file upload with same file uploaded twice.""" + headers_user1 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user1, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user1, + ) + + assert response + assert 200 == response.status_code + assert {'error'} == set(response.json.keys()) + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'file exists' == response.json['error']['reason'] + + +@pytest.mark.service +def test_file_upload_no_auth(svc_client): + """Check failed file upload.""" + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + ) + + assert response + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +def test_file_upload_with_users(svc_client): + """Check successful file upload and listing based on user auth header.""" + headers_user1 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + headers_user2 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers_user1 + ) + + assert {'result'} == set(response.json.keys()) + + file_id = response.json['result']['files'][0]['file_id'] + assert file_id + assert 200 == response.status_code + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers_user2 + ) + + assert response + assert {'result'} == set(response.json.keys()) + + response = svc_client.get('/cache/files-list', headers=headers_user1) + + assert response + + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + file = response.json['result']['files'][0] + assert file_id == file['file_id'] + assert 0 < file['file_size'] + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_no_auth(svc_client): + """Check error on cloning of remote repository.""" + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload) + ) + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer notatoken', + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_with_auth(svc_client): + """Check cloning of remote repository.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_multiple(svc_client): + """Check multiple cloning of remote repository.""" + project_ids = [] + + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + last_pid = response.json['result']['project_id'] + + response = svc_client.get('/cache/project-list', headers=headers) + + assert response + assert {'result'} == set(response.json.keys()) + + pids = [p['project_id'] for p in response.json['result']['projects']] + assert last_pid in pids + + for inserted in project_ids: + assert inserted['project_id'] not in pids + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_list_view_errors(svc_client): + """Check cache state of cloned projects with no headers.""" + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + assert response + assert {'result'} == set(response.json.keys()) + + assert isinstance( + uuid.UUID(response.json['result']['project_id']), uuid.UUID + ) + + response = svc_client.get( + '/cache/project-list', + # no auth headers, expected error + ) + assert response + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + response = svc_client.get('/cache/project-list', headers=headers) + assert response + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['projects']) + + project = response.json['result']['projects'][0] + assert isinstance(uuid.UUID(project['project_id']), uuid.UUID) + assert isinstance(GitURL.parse(project['git_url']), GitURL) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_invalid_headers(svc_client): + """Check cache state of cloned projects with invalid headers.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Not Sam', + 'Renku-User-Email': 'not@sam.io', + 'Authorization': 'Bearer not-a-token', + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', + data=json.dumps(payload), + headers=headers, + ) + assert response + assert {'result'} == set(response.json.keys()) + + response = svc_client.get( + '/cache/project-list', + # no auth headers, expected error + ) + + assert response + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + response = svc_client.get('/cache/project-list', headers=headers) + assert response + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['projects']) + + +@pytest.mark.service +def test_upload_zip_unpack_archive(datapack_zip, svc_client_with_repo): + """Upload zip archive with unpack.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_zip_archive(datapack_zip, svc_client_with_repo): + """Upload zip archive.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': False, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_tar_unpack_archive(datapack_tar, svc_client_with_repo): + """Upload zip archive with unpack.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_tar_archive(datapack_tar, svc_client_with_repo): + """Upload zip archive.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': False, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_field_upload_resp_fields(datapack_tar, svc_client_with_repo): + """Check response fields.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + assert { + 'content_type', + 'file_id', + 'file_name', + 'file_size', + 'is_archive', + 'timestamp', + 'is_archive', + 'unpack_archive', + 'relative_path', + } == set(response.json['result']['files'][0].keys()) + + assert not response.json['result']['files'][0]['is_archive'] + assert not response.json['result']['files'][0]['unpack_archive'] + + rel_path = response.json['result']['files'][0]['relative_path'] + assert rel_path.startswith(datapack_tar.name) and 'unpacked' in rel_path diff --git a/tests/service/test_dataset_views.py b/tests/service/test_dataset_views.py new file mode 100644 index 0000000000..60780daa6b --- /dev/null +++ b/tests/service/test_dataset_views.py @@ -0,0 +1,547 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service dataset view tests.""" +import io +import json +import uuid + +import pytest + +from renku.service.config import INVALID_HEADERS_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE, RENKU_EXCEPTION_ERROR_CODE + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view(svc_client_with_repo): + """Create new dataset successfully.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view_dataset_exists(svc_client_with_repo): + """Create new dataset which already exists.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydataset', + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + assert RENKU_EXCEPTION_ERROR_CODE == response.json['error']['code'] + assert 'Dataset exists' in response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view_unknown_param(svc_client_with_repo): + """Create new dataset by specifying unknown parameters.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin' + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert {'remote_name'} == set(response.json['error']['reason'].keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_with_no_identity(svc_client_with_repo): + """Create new dataset with no identification provided.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin', + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers={'Content-Type': headers['Content-Type']} + # no user identity, expect error + ) + + assert response + assert {'error'} == response.json.keys() + + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view_with_no_identity(svc_client_with_repo): + """Check identity error raise in dataset add.""" + svc_client, headers, project_id = svc_client_with_repo + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin', + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers={'Content-Type': headers['Content-Type']} + # no user identity, expect error + ) + assert response + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view(svc_client_with_repo): + """Check adding of uploaded file to dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + query_string={'override_existing': True}, + headers=headers + ) + + assert response + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + file_id = response.json['result']['files'][0]['file_id'] + assert isinstance(uuid.UUID(file_id), uuid.UUID) + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + 'create_dataset': True, + 'files': [{ + 'file_id': file_id, + }, ] + } + headers['Content-Type'] = content_type + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'project_id', + 'files'} == set(response.json['result'].keys()) + + assert 1 == len(response.json['result']['files']) + assert file_id == response.json['result']['files'][0]['file_id'] + + +@pytest.mark.service +@pytest.mark.integration +def test_list_datasets_view(svc_client_with_repo): + """Check listing of existing datasets.""" + svc_client, headers, project_id = svc_client_with_repo + + params = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'datasets'} == set(response.json['result'].keys()) + assert 0 != len(response.json['result']['datasets']) + assert {'identifier', 'name', 'version', + 'created'} == set(response.json['result']['datasets'][0].keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_list_datasets_view_no_auth(svc_client_with_repo): + """Check listing of existing datasets with no auth.""" + svc_client, headers, project_id = svc_client_with_repo + + params = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_create_and_list_datasets_view(svc_client_with_repo): + """Create and list created dataset.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + params_list = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params_list, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'datasets'} == set(response.json['result'].keys()) + assert 0 != len(response.json['result']['datasets']) + assert {'identifier', 'name', 'version', + 'created'} == set(response.json['result']['datasets'][0].keys()) + + assert payload['dataset_name'] in [ + ds['name'] for ds in response.json['result']['datasets'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_list_dataset_files(svc_client_with_repo): + """Check listing of dataset files""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + file_name = '{0}'.format(uuid.uuid4().hex) + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), file_name), ), + query_string={'override_existing': True}, + headers=headers + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + file_id = response.json['result']['files'][0]['file_id'] + assert isinstance(uuid.UUID(file_id), uuid.UUID) + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'files': [{ + 'file_id': file_id + }, ], + } + headers['Content-Type'] = content_type + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert file_id == response.json['result']['files'][0]['file_id'] + + params = { + 'project_id': project_id, + 'dataset_name': 'mydata', + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_name in [ + file['name'] for file in response.json['result']['files'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): + """Upload archive and add it to a dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + file_id = file_['file_id'] + assert file_id + + file_ = response.json['result']['files'][0] + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + headers['Content-Type'] = content_type + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'files': [{ + 'file_id': file_['file_id'] + }, ] + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert file_['file_id'] == response.json['result']['files'][0]['file_id'] + + params = { + 'project_id': project_id, + 'dataset_name': 'mydata', + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_['file_name'] in [ + file['name'] for file in response.json['result']['files'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): + """Upload archive and add its contents to a dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + file_id = file_['file_id'] + assert file_id + + files = [{ + 'file_id': file_['file_id'] + } for file_ in response.json['result']['files']] + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + headers['Content-Type'] = content_type + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + payload = { + 'project_id': project_id, + 'dataset_name': payload['dataset_name'], + 'files': files, + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert files == response.json['result']['files'] + + params = { + 'project_id': project_id, + 'dataset_name': payload['dataset_name'], + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_['file_name'] in [ + file['name'] for file in response.json['result']['files'] + ] diff --git a/tests/service/test_exceptions.py b/tests/service/test_exceptions.py new file mode 100644 index 0000000000..d5bd5b4c1f --- /dev/null +++ b/tests/service/test_exceptions.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service exception tests for all endpoints.""" +import pytest + +from renku.service.config import INVALID_HEADERS_ERROR_CODE + + +@pytest.mark.service +def test_allowed_methods_exc(service_allowed_endpoint): + """Check allowed methods for every endpoint.""" + methods, request, svc_client = service_allowed_endpoint + + method = request['allowed_method'] + if method == 'GET': # if GET remove sister method HEAD + methods.pop(method) + methods.pop('HEAD') + else: + methods.pop(method) + + for method, fn in methods.items(): + response = fn(request['url']) + assert 405 == response.status_code + + +@pytest.mark.service +def test_auth_headers_exc(service_allowed_endpoint): + """Check correct headers for every endpoint.""" + methods, request, svc_client = service_allowed_endpoint + + method = request['allowed_method'] + if method == 'GET': # if GET remove sister method HEAD + client_method = methods.pop(method) + methods.pop('HEAD') + else: + client_method = methods.pop(method) + + response = client_method( + request['url'], + headers=request['headers'], + ) + + assert 200 == response.status_code + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason']