From 3d27841da013554823476ea761e0e15f7a00c2df Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:55:26 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20Update=20version=20and=20testing?= =?UTF-8?q?=20for=20release=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove new functionality * change version * add staging branch to client-server compatibility * update API * start changing docu * update tutorial * update documentation * make sure master is up to date with changes to staging * skip notebook tests if not correct versions * minor changes to make notebook tests pass * fix * cosmetic change * update basic tutorial * attempt to add workflow for publishing both packages * go back to 3.9 * introduce dev features flag * change e2e tests * cosmetic change * add dev feature flag to pytest ini * improve decorator * improve logging * naming change * change naming conventions for e2e tests * ensure to record dev features * minor change * fix install script * fix run script * improve client ref * update compatibility json * minor change --- .github/workflows/publish-python-client.yml | 108 +++- clients/python/client/osparc/__init__.py | 41 +- clients/python/client/osparc/_files_api.py | 295 +++++----- clients/python/client/osparc/_solvers_api.py | 70 ++- clients/python/client/osparc/_utils.py | 5 + ...orial.ipynb => BasicTutorial_v0.5.0.ipynb} | 2 +- .../python/docs/BasicTutorial_v0.6.0.ipynb | 532 ++++++++++++++++++ clients/python/docs/v0.5.0/README.md | 2 +- clients/python/docs/v0.6.0/README.md | 8 + clients/python/requirements/test.txt | 1 + clients/python/test/e2e/_utils.py | 30 + clients/python/test/e2e/ci/_data_classes.py | 71 +-- .../python/test/e2e/ci/generate_pytest_ini.py | 3 + .../e2e/ci/install_osparc_python_client.bash | 18 +- .../test/e2e/ci/preprocess_client_config.bash | 38 +- .../test/e2e/ci/run_e2e_python_tests.bash | 5 - .../e2e/data/server_client_compatibility.json | 51 +- clients/python/test/e2e/test_files_api.py | 12 +- clients/python/test/e2e/test_notebooks.py | 23 +- clients/python/test/e2e/test_solvers_api.py | 8 +- docs/navbar.md | 1 + 21 files changed, 997 insertions(+), 327 deletions(-) rename clients/python/docs/{BasicTutorial.ipynb => BasicTutorial_v0.5.0.ipynb} (99%) create mode 100755 clients/python/docs/BasicTutorial_v0.6.0.ipynb create mode 100644 clients/python/docs/v0.6.0/README.md create mode 100644 clients/python/test/e2e/_utils.py diff --git a/.github/workflows/publish-python-client.yml b/.github/workflows/publish-python-client.yml index c8ac1456..fef22727 100644 --- a/.github/workflows/publish-python-client.yml +++ b/.github/workflows/publish-python-client.yml @@ -14,8 +14,8 @@ jobs: build-n-publish: runs-on: ubuntu-latest outputs: - wheel: ${{ steps.find-wheel.outputs.osparc-wheel-path }} - dist-dir: ${{ steps.find-wheel.outputs.osparc-auto-wheel-dir }} + osparc: ${{ steps.find-wheel.outputs.osparc-wheel }} + osparc_client: ${{ steps.find-wheel.outputs.osparc_client-wheel }} steps: - uses: actions/checkout@v3 with: @@ -35,34 +35,16 @@ jobs: - name: Determine wheel artifact id: find-wheel run: | - OSPARC_WHEEL=$(ls clients/python/artifacts/dist/osparc-*.whl) - OSPARC_AUTO_GENERATED_WHEEL=$(ls clients/python/artifacts/dist/osparc_client*.whl) - echo "osparc-wheel-path=${OSPARC_WHEEL}" >> $GITHUB_OUTPUT - echo "osparc-wheel-dir=$(dirname ${OSPARC_WHEEL})" >> $GITHUB_OUTPUT - echo "osparc-wheel-name=$(basename ${OSPARC_WHEEL})" >> $GITHUB_OUTPUT - echo "osparc-auto-wheel-path=${OSPARC_AUTO_GENERATED_WHEEL}" >> $GITHUB_OUTPUT - echo "osparc-auto-wheel-dir=$(dirname ${OSPARC_AUTO_GENERATED_WHEEL})" >> $GITHUB_OUTPUT - echo "osparc-auto-wheel-name=$(basename ${OSPARC_AUTO_GENERATED_WHEEL})" >> $GITHUB_OUTPUT + cd clients/python/artifacts/dist + OSPARC_WHEEL=$(ls osparc-*.whl) + OSPARC_CLIENT_WHEEL=$(ls osparc_client*.whl) + echo "osparc-wheel=${OSPARC_WHEEL}" >> $GITHUB_OUTPUT + echo "osparc_client-wheel=${OSPARC_CLIENT_WHEEL}" >> $GITHUB_OUTPUT - name: Upload wheels uses: actions/upload-artifact@v3 with: name: osparc_python_wheels path: clients/python/artifacts/dist/ - - name: Publish package to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - with: - password: ${{ secrets.TEST_PYPI_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - verbose: true - packages-dir: ${{ steps.find-wheel.outputs.wheel-dir }} - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - with: - password: ${{ secrets.PYPI_TOKEN }} - verbose: true - packages-dir: ${{ steps.find-wheel.outputs.wheel-dir }} test-20-04: name: python ${{ matrix.python-version }} ubuntu-20.04 @@ -87,17 +69,17 @@ jobs: key: ${{ runner.os }}-pip-{{ matrix.python-version }} restore-keys: | ${{ runner.os }}-pip - - name: Donwload client + - name: Download wheels uses: actions/download-artifact@v3 with: name: osparc_python_wheels path: clients/python/artifacts/dist/ - name: Install and Test run: | - make devenv + python -m venv .venv source .venv/bin/activate python -m pip install pytest - python -m pip install ${{needs.build-n-publish.outputs.wheel}} --find-links=${{needs.build-n-publish.outputs.dist-dir}} + python -m pip install clients/python/artifacts/dist/${{needs.build-n-publish.outputs.osparc}} --find-links=clients/python/artifacts/dist cd clients/python make install-test pytest -v --ignore=/artifacts/client --ignore=test/e2e @@ -123,17 +105,81 @@ jobs: key: ${{ runner.os }}-pip-{{ matrix.python-version }} restore-keys: | ${{ runner.os }}-pip - - name: Donwload client + - name: Download wheels uses: actions/download-artifact@v3 with: name: osparc_python_wheels path: clients/python/artifacts/dist/ - name: Install and Test run: | - make devenv + python -m venv .venv source .venv/bin/activate python -m pip install pytest - python -m pip install ${{needs.build-n-publish.outputs.wheel}} --find-links=${{needs.build-n-publish.outputs.dist-dir}} + python -m pip install clients/python/artifacts/dist/${{needs.build-n-publish.outputs.osparc}} --find-links=clients/python/artifacts/dist cd clients/python make install-test pytest -v --ignore=/artifacts/client --ignore=test/e2e + + publish-osparc_client-to-pypi: + name: Publish osparc_client wheel + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + runs-on: ubuntu-latest + needs: [build-n-publish, test-20-04, test-latest] + environment: + name: pypi + url: https://pypi.org/p/osparc_client + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: osparc_python_wheels + path: dist/ + - name: Remove osparc wheel + run: rm -f dist/${{needs.build-n-publish.outputs.osparc}} + - name: Publish osparc_client to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + verbose: true + packages-dir: dist/ + - name: Publish osparc_client to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + verbose: true + packages-dir: dist/ + + publish-osparc-to-pypi: + name: Publish osparc wheel + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + runs-on: ubuntu-latest + needs: [build-n-publish, test-20-04, test-latest] + environment: + name: pypi + url: https://pypi.org/p/osparc + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: osparc_python_wheels + path: dist/ + - name: Remove osparc_client wheel + run: rm -f dist/${{needs.build-n-publish.outputs.osparc_client}} + - name: Publish osparc to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + verbose: true + packages-dir: dist/ + - name: Publish osparc to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + verbose: true + packages-dir: dist/ diff --git a/clients/python/client/osparc/__init__.py b/clients/python/client/osparc/__init__.py index dafd1075..5e56c8c1 100644 --- a/clients/python/client/osparc/__init__.py +++ b/clients/python/client/osparc/__init__.py @@ -1,7 +1,4 @@ -""" -0.6.0 osparc client -""" -from typing import Tuple +from typing import List, Tuple import nest_asyncio from osparc_client import ( # APIs; API client; models @@ -18,15 +15,11 @@ HTTPValidationError, Job, JobInputs, - JobMetadata, - JobMetadataUpdate, JobOutputs, JobStatus, - Links, Meta, MetaApi, OnePageSolverPort, - OnePageStudyPort, OpenApiException, Profile, ProfileUpdate, @@ -35,9 +28,6 @@ from osparc_client import ( # APIs; API client; models Solver, SolverPort, - StudiesApi, - Study, - StudyPort, UserRoleEnum, UsersApi, UsersGroup, @@ -48,15 +38,26 @@ from ._files_api import FilesApi from ._info import openapi from ._solvers_api import SolversApi -from ._utils import PaginationGenerator +from ._utils import dev_features_enabled nest_asyncio.apply() # allow to run coroutines via asyncio.run(coro) -__all__: Tuple[str, ...] = ( - # imports from osparc_client +dev_features: List[str] = [] +if dev_features_enabled(): + dev_features = [ + "PaginationGenerator", + "StudiesApi", + "StudyPort", + "Study", + "JobMetadataUpdate", + "Links", + "JobMetadata", + "OnePageStudyPort", + ] + +__all__: Tuple[str, ...] = tuple(dev_features) + ( "__version__", "FilesApi", - "PaginationGenerator", "MetaApi", "SolversApi", "UsersApi", @@ -83,16 +84,8 @@ "ApiValueError", "ApiKeyError", "ApiException", - "StudiesApi", "OnePageSolverPort", - "StudyPort", - "Study", - "JobMetadataUpdate", - "Links", "SolverPort", - "JobMetadata", "ErrorGet", - "OnePageStudyPort", - # imports from osparc "openapi", -) +) # type: ignore diff --git a/clients/python/client/osparc/_files_api.py b/clients/python/client/osparc/_files_api.py index 392bd667..d860ec01 100644 --- a/clients/python/client/osparc/_files_api.py +++ b/clients/python/client/osparc/_files_api.py @@ -20,160 +20,171 @@ from . import ApiClient, File from ._http_client import AsyncHttpClient -from ._utils import PaginationGenerator, compute_sha256, file_chunk_generator +from ._utils import ( + PaginationGenerator, + compute_sha256, + dev_features_enabled, + file_chunk_generator, +) class FilesApi(_FilesApi): """Class for interacting with files""" - def __init__(self, api_client: Optional[ApiClient] = None): - """Construct object - - Args: - api_client (ApiClient, optinal): osparc.ApiClient object - """ - super().__init__(api_client) - self._super = super(FilesApi, self) - user: Optional[str] = self.api_client.configuration.username - passwd: Optional[str] = self.api_client.configuration.password - self._auth: Optional[httpx.BasicAuth] = ( - httpx.BasicAuth(username=user, password=passwd) - if (user is not None and passwd is not None) - else None - ) - - def download_file( - self, file_id: str, *, destination_folder: Optional[Path] = None - ) -> str: - if destination_folder is not None and not destination_folder.is_dir(): - raise RuntimeError( - f"destination_folder: {destination_folder} must be a directory" + if dev_features_enabled(): + + def __init__(self, api_client: Optional[ApiClient] = None): + """Construct object + + Args: + api_client (ApiClient, optinal): osparc.ApiClient object + """ + super().__init__(api_client) + self._super = super(FilesApi, self) + user: Optional[str] = self.api_client.configuration.username + passwd: Optional[str] = self.api_client.configuration.password + self._auth: Optional[httpx.BasicAuth] = ( + httpx.BasicAuth(username=user, password=passwd) + if (user is not None and passwd is not None) + else None ) - downloaded_file: Path = Path(super().download_file(file_id)) - if destination_folder is not None: - dest_file: Path = destination_folder / downloaded_file.name - while dest_file.is_file(): - new_name = ( - downloaded_file.stem - + "".join(random.choices(string.ascii_letters, k=8)) - + downloaded_file.suffix + + def download_file( + self, file_id: str, *, destination_folder: Optional[Path] = None + ) -> str: + if destination_folder is not None and not destination_folder.is_dir(): + raise RuntimeError( + f"destination_folder: {destination_folder} must be a directory" ) - dest_file = destination_folder / new_name - shutil.move(downloaded_file, dest_file) - downloaded_file = dest_file - return str(downloaded_file.resolve()) - - def upload_file(self, file: Union[str, Path]): - return asyncio.run(self.upload_file_async(file=file)) - - async def upload_file_async(self, file: Union[str, Path]) -> File: - if isinstance(file, str): - file = Path(file) - if not file.is_file(): - raise RuntimeError(f"{file} is not a file") - checksum: str = compute_sha256(file) - for file_result in self._search_files(sha256_checksum=checksum): - if file_result.filename == file.name: - # if a file has the same sha256 checksum - # and name they are considered equal - return file_result - client_file: ClientFile = ClientFile( - filename=file.name, - filesize=file.stat().st_size, - sha256_checksum=checksum, - ) - client_upload_schema: ClientFileUploadData = self._super.get_upload_links( - client_file=client_file - ) - chunk_size: int = client_upload_schema.upload_schema.chunk_size - links: FileUploadData = client_upload_schema.upload_schema.links - url_iter: Iterator[Tuple[int, str]] = enumerate( - iter(client_upload_schema.upload_schema.urls), start=1 - ) - n_urls: int = len(client_upload_schema.upload_schema.urls) - if n_urls < math.ceil(file.stat().st_size / chunk_size): - raise RuntimeError( - "Did not receive sufficient number of upload URLs from the server." + downloaded_file: Path = Path(super().download_file(file_id)) + if destination_folder is not None: + dest_file: Path = destination_folder / downloaded_file.name + while dest_file.is_file(): + new_name = ( + downloaded_file.stem + + "".join(random.choices(string.ascii_letters, k=8)) + + downloaded_file.suffix + ) + dest_file = destination_folder / new_name + shutil.move(downloaded_file, dest_file) + downloaded_file = dest_file + return str(downloaded_file.resolve()) + + def upload_file(self, file: Union[str, Path]): + return asyncio.run(self.upload_file_async(file=file)) + + async def upload_file_async(self, file: Union[str, Path]) -> File: + if isinstance(file, str): + file = Path(file) + if not file.is_file(): + raise RuntimeError(f"{file} is not a file") + checksum: str = compute_sha256(file) + for file_result in self._search_files(sha256_checksum=checksum): + if file_result.filename == file.name: + # if a file has the same sha256 checksum + # and name they are considered equal + return file_result + client_file: ClientFile = ClientFile( + filename=file.name, + filesize=file.stat().st_size, + sha256_checksum=checksum, + ) + client_upload_schema: ClientFileUploadData = self._super.get_upload_links( + client_file=client_file + ) + chunk_size: int = client_upload_schema.upload_schema.chunk_size + links: FileUploadData = client_upload_schema.upload_schema.links + url_iter: Iterator[Tuple[int, str]] = enumerate( + iter(client_upload_schema.upload_schema.urls), start=1 ) + n_urls: int = len(client_upload_schema.upload_schema.urls) + if n_urls < math.ceil(file.stat().st_size / chunk_size): + raise RuntimeError( + "Did not receive sufficient number of upload URLs from the server." + ) - uploaded_parts: list[UploadedPart] = [] - print("- uploading chunks...") - async with AsyncHttpClient() as session: - async for chunck, size in tqdm( - file_chunk_generator(file, chunk_size), total=n_urls - ): - index, url = next(url_iter) - uploaded_parts.append( - await self._upload_chunck( - http_client=session, - chunck=chunck, - chunck_size=size, - upload_link=url, - index=index, + uploaded_parts: list[UploadedPart] = [] + print("- uploading chunks...") + async with AsyncHttpClient() as session: + async for chunck, size in tqdm( + file_chunk_generator(file, chunk_size), total=n_urls + ): + index, url = next(url_iter) + uploaded_parts.append( + await self._upload_chunck( + http_client=session, + chunck=chunck, + chunck_size=size, + upload_link=url, + index=index, + ) ) + + async with AsyncHttpClient( + request_type="post", + url=links.abort_upload, + base_url=self.api_client.configuration.host, + follow_redirects=True, + auth=self._auth, + ) as session: + print( + "- completing upload (this might take a couple of minutes)..." + ) + file: File = await self._complete_multipart_upload( + session, links.complete_upload, client_file, uploaded_parts + ) + print("- file upload complete") + return file + + async def _complete_multipart_upload( + self, + http_client: AsyncClient, + complete_link: str, + client_file: ClientFile, + uploaded_parts: List[UploadedPart], + ) -> File: + complete_payload = BodyCompleteMultipartUploadV0FilesFileIdCompletePost( + client_file=client_file, + uploaded_parts=FileUploadCompletionBody(parts=uploaded_parts), + ) + response: Response = await http_client.post( + complete_link, + json=complete_payload.to_dict(), + ) + response.raise_for_status() + payload: dict[str, Any] = response.json() + return File(**payload) + + async def _upload_chunck( + self, + http_client: AsyncClient, + chunck: bytes, + chunck_size: int, + upload_link: str, + index: int, + ) -> UploadedPart: + response: Response = await http_client.put( + upload_link, + content=chunck, + headers={"Content-Length": f"{chunck_size}"}, + ) + response.raise_for_status() + assert response.headers # nosec + assert "Etag" in response.headers # nosec + etag: str = json.loads(response.headers["Etag"]) + return UploadedPart(number=index, e_tag=etag) + + def _search_files( + self, file_id: Optional[str] = None, sha256_checksum: Optional[str] = None + ) -> PaginationGenerator: + def pagination_method(): + return super(FilesApi, self).search_files_page( + file_id=file_id, sha256_checksum=sha256_checksum ) - async with AsyncHttpClient( - request_type="post", - url=links.abort_upload, + return PaginationGenerator( + first_page_callback=pagination_method, + api_client=self.api_client, base_url=self.api_client.configuration.host, - follow_redirects=True, auth=self._auth, - ) as session: - print("- completing upload (this might take a couple of minutes)...") - file: File = await self._complete_multipart_upload( - session, links.complete_upload, client_file, uploaded_parts - ) - print("- file upload complete") - return file - - async def _complete_multipart_upload( - self, - http_client: AsyncClient, - complete_link: str, - client_file: ClientFile, - uploaded_parts: List[UploadedPart], - ) -> File: - complete_payload = BodyCompleteMultipartUploadV0FilesFileIdCompletePost( - client_file=client_file, - uploaded_parts=FileUploadCompletionBody(parts=uploaded_parts), - ) - response: Response = await http_client.post( - complete_link, - json=complete_payload.to_dict(), - ) - response.raise_for_status() - payload: dict[str, Any] = response.json() - return File(**payload) - - async def _upload_chunck( - self, - http_client: AsyncClient, - chunck: bytes, - chunck_size: int, - upload_link: str, - index: int, - ) -> UploadedPart: - response: Response = await http_client.put( - upload_link, content=chunck, headers={"Content-Length": f"{chunck_size}"} - ) - response.raise_for_status() - assert response.headers # nosec - assert "Etag" in response.headers # nosec - etag: str = json.loads(response.headers["Etag"]) - return UploadedPart(number=index, e_tag=etag) - - def _search_files( - self, file_id: Optional[str] = None, sha256_checksum: Optional[str] = None - ) -> PaginationGenerator: - def pagination_method(): - return super(FilesApi, self).search_files_page( - file_id=file_id, sha256_checksum=sha256_checksum ) - - return PaginationGenerator( - first_page_callback=pagination_method, - api_client=self.api_client, - base_url=self.api_client.configuration.host, - auth=self._auth, - ) diff --git a/clients/python/client/osparc/_solvers_api.py b/clients/python/client/osparc/_solvers_api.py index 984dce92..08da5ede 100644 --- a/clients/python/client/osparc/_solvers_api.py +++ b/clients/python/client/osparc/_solvers_api.py @@ -1,10 +1,11 @@ -from typing import Optional +from typing import List, Optional import httpx +from osparc_client import OnePageSolverPort, SolverPort from osparc_client import SolversApi as _SolversApi from . import ApiClient -from ._utils import PaginationGenerator +from ._utils import PaginationGenerator, dev_features_enabled class SolversApi(_SolversApi): @@ -16,7 +17,8 @@ def __init__(self, api_client: Optional[ApiClient] = None): Args: api_client (ApiClient, optinal): osparc.ApiClient object """ - super().__init__(api_client) + self._super: _SolversApi = super() + self._super.__init__(api_client) user: Optional[str] = self.api_client.configuration.username passwd: Optional[str] = self.api_client.configuration.password self._auth: Optional[httpx.BasicAuth] = ( @@ -25,34 +27,42 @@ def __init__(self, api_client: Optional[ApiClient] = None): else None ) - def get_jobs_page(self, solver_key: str, version: str) -> None: - """Method only for internal use""" - raise NotImplementedError("This method is only for internal use") + def list_solver_ports(self, solver_key: str, version: str) -> List[SolverPort]: + page: OnePageSolverPort = self._super.list_solver_ports( + solver_key=solver_key, version=version + ) # type: ignore + return page.items if page.items else [] - def jobs(self, solver_key: str, version: str) -> PaginationGenerator: - """Returns an iterator through which one can iterate over - all Jobs submitted to the solver + if dev_features_enabled(): - Args: - solver_key (str): The solver key - version (str): The solver version - limit (int, optional): the limit of a single page - offset (int, optional): the offset of the first element to return - - Returns: - PaginationGenerator: A generator whose elements are the Jobs submitted - to the solver and the total number of jobs the iterator can yield - (its "length") - """ + def get_jobs_page(self, solver_key: str, version: str) -> None: + """Method only for internal use""" + raise NotImplementedError("This method is only for internal use") - def pagination_method(): - return super(SolversApi, self).get_jobs_page( - solver_key=solver_key, version=version, limit=20, offset=0 - ) + def jobs(self, solver_key: str, version: str) -> PaginationGenerator: + """Returns an iterator through which one can iterate over + all Jobs submitted to the solver - return PaginationGenerator( - first_page_callback=pagination_method, - api_client=self.api_client, - base_url=self.api_client.configuration.host, - auth=self._auth, - ) + Args: + solver_key (str): The solver key + version (str): The solver version + limit (int, optional): the limit of a single page + offset (int, optional): the offset of the first element to return + + Returns: + PaginationGenerator: A generator whose elements are the Jobs submitted + to the solver and the total number of jobs the iterator can yield + (its "length") + """ + + def pagination_method(): + return super(SolversApi, self).get_jobs_page( + solver_key=solver_key, version=version, limit=20, offset=0 + ) + + return PaginationGenerator( + first_page_callback=pagination_method, + api_client=self.api_client, + base_url=self.api_client.configuration.host, + auth=self._auth, + ) diff --git a/clients/python/client/osparc/_utils.py b/clients/python/client/osparc/_utils.py index fdfda671..934cf4de 100644 --- a/clients/python/client/osparc/_utils.py +++ b/clients/python/client/osparc/_utils.py @@ -1,5 +1,6 @@ import asyncio import hashlib +import os from pathlib import Path from typing import AsyncGenerator, Callable, Generator, Optional, Tuple, TypeVar, Union @@ -107,3 +108,7 @@ def compute_sha256(file: Path) -> str: break sha256.update(data) return sha256.hexdigest() + + +def dev_features_enabled() -> bool: + return os.environ.get("OSPARC_DEV_FEATURES_ENABLED") == "1" diff --git a/clients/python/docs/BasicTutorial.ipynb b/clients/python/docs/BasicTutorial_v0.5.0.ipynb similarity index 99% rename from clients/python/docs/BasicTutorial.ipynb rename to clients/python/docs/BasicTutorial_v0.5.0.ipynb index b5112f08..a61fedf9 100755 --- a/clients/python/docs/BasicTutorial.ipynb +++ b/clients/python/docs/BasicTutorial_v0.5.0.ipynb @@ -520,7 +520,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.0" + "version": "3.10.10" } }, "nbformat": 4, diff --git a/clients/python/docs/BasicTutorial_v0.6.0.ipynb b/clients/python/docs/BasicTutorial_v0.6.0.ipynb new file mode 100755 index 00000000..3befca2a --- /dev/null +++ b/clients/python/docs/BasicTutorial_v0.6.0.ipynb @@ -0,0 +1,532 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "f15de720", + "metadata": {}, + "source": [ + "# Basic Tutorial\n", + "\n", + "\n", + "\n", + "## Installation\n", + "Install the python client and check the installation as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09653e21", + "metadata": { + "attributes": { + "classes": [ + "command" + ], + "id": "" + } + }, + "outputs": [], + "source": [ + "import importlib\n", + "if importlib.util.find_spec('osparc') is not None:\n", + " ! pip install osparc\n", + "! python -c \"import osparc; print(osparc.__version__)\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "34c8833d", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "To setup the client, we need to provide a username and password to the configuration. These can be obtained in the UI under [Preferences > API Settings > API Keys](https://docs.osparc.io/#/docs/platform_introduction/user_setup/security_details?id=generating-o%c2%b2s%c2%b2parc-tokens). Use the *API key* as username and the *API secret* as password. For security reasons, you should not write these values in your script but instead set them up via environment variables or read them from a separate file. In this example, we use environment variables which will be referred to as \"OSPARC_API_KEY\" and \"OSPARC_API_SECRET\" for the rest of the tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85ac5228", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import os\n", + "from osparc import Configuration\n", + "\n", + "cfg = Configuration(\n", + " host=os.environ[\"OSPARC_API_HOST\"],\n", + " username=os.environ[\"OSPARC_API_KEY\"],\n", + " password=os.environ[\"OSPARC_API_SECRET\"],\n", + ")\n", + "print(cfg.host)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "630c5926", + "metadata": {}, + "source": [ + "The configuration can now be used to create an instance of the API client. The API client is responsible of the communication with the osparc platform\n", + "\n", + "\n", + "The functions in the [osparc API](https://api.osparc.io/dev/doc#/) are grouped into sections such as *meta*, *users*, *files* or *solvers*. Each section address a different resource of the platform.\n", + "\n", + "\n", + "\n", + "For example, the *users* section includes functions about the user (i.e. you) and can be accessed initializing a ``UsersApi``:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29337833", + "metadata": {}, + "outputs": [], + "source": [ + "from osparc import ApiClient, UsersApi\n", + "\n", + "with ApiClient(cfg) as api_client:\n", + "\n", + " users_api = UsersApi(api_client)\n", + "\n", + " profile = users_api.get_my_profile()\n", + " print(profile)\n", + "\n", + " #\n", + " # {'first_name': 'foo',\n", + " # 'gravatar_id': 'aa33fssec77ea434c2ea4fb92d0fd379e',\n", + " # 'groups': {'all': {'description': 'all users',\n", + " # 'gid': '1',\n", + " # 'label': 'Everyone'},\n", + " # 'me': {'description': 'primary group',\n", + " # 'gid': '2',\n", + " # 'label': 'foo'},\n", + " # 'organizations': []},\n", + " # 'last_name': '',\n", + " # 'login': 'foo@itis.swiss',\n", + " # 'role': 'USER'}\n", + " #" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6912889e", + "metadata": {}, + "source": [ + "## Solvers Workflow\n", + "\n", + "The osparc API can be used to execute any computational service published in the platform. This means that any computational service listed in the UI under the [Services Tab](https://docs.osparc.io/#/docs/platform_introduction/services) is accessible from the API. Note that computational services are denoted as *solvers* in the API for convenience, but they refer to the same concept.\n", + "\n", + "\n", + "Let's use the sleepers computational service to illustrate a typical workflow. The sleepers computational service is a very basic service that simply waits (i.e. *sleeps*) a given time before producing some outputs. It takes as input one natural number, an optional text file input that contains another natural number and a boolean in the form of a checkbox. It also provides two outputs: one natural number and a file containing a single natural number." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "398bcd09", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "from pathlib import Path\n", + "from zipfile import ZipFile\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "import osparc\n", + "\n", + "CLIENT_VERSION = tuple(map(int, osparc.__version__.split(\".\")))\n", + "assert CLIENT_VERSION >= (0, 4, 3)\n", + "\n", + "Path(\"file_with_number.txt\").write_text(\"3\")\n", + "\n", + "with osparc.ApiClient(cfg) as api_client:\n", + "\n", + " files_api = osparc.FilesApi(api_client)\n", + " input_file: osparc.File = files_api.upload_file(file=\"file_with_number.txt\")\n", + "\n", + " solver_key: str = \"simcore/services/comp/itis/sleeper\"\n", + " solver_version: str = \"2.0.2\"\n", + " \n", + " solvers_api = osparc.SolversApi(api_client)\n", + " solver: osparc.Solver = solvers_api.get_solver_release(solver_key=solver_key, version=solver_version)\n", + "\n", + " solver_ports = solvers_api.list_solver_ports(solver.id, solver.version)\n", + " print(f\"solver_ports: {solver_ports}\")\n", + "\n", + " job: osparc.Job = solvers_api.create_job(\n", + " solver.id,\n", + " solver.version,\n", + " osparc.JobInputs(\n", + " {\n", + " \"input_3\": 0,\n", + " \"input_2\": 3.0,\n", + " \"input_1\": input_file,\n", + " }\n", + " ),\n", + " )\n", + "\n", + " status: osparc.JobStatus = solvers_api.start_job(solver.id, solver.version, job.id)\n", + " while not status.stopped_at:\n", + " time.sleep(3)\n", + " status = solvers_api.inspect_job(solver.id, solver.version, job.id)\n", + " print(\"Solver progress\", f\"{status.progress}/100\", flush=True)\n", + " #\n", + " # Solver progress 0/100\n", + " # Solver progress 100/100\n", + "\n", + " outputs: osparc.JobOutputs = solvers_api.get_job_outputs(solver.id, solver.version, job.id)\n", + "\n", + " print(f\"Job {outputs.job_id} got these results:\")\n", + " for output_name, result in outputs.results.items():\n", + " print(output_name, \"=\", result)\n", + "\n", + " #\n", + " # Job 19fc28f7-46fb-4e96-9129-5e924801f088 got these results:\n", + " #\n", + " # output_1 = {'checksum': '859fda0cb82fc4acb4686510a172d9a9-1',\n", + " # 'content_type': 'text/plain',\n", + " # 'filename': 'single_number.txt',\n", + " # 'id': '9fb4f70e-3589-3e9e-991e-3059086c3aae'}\n", + " # output_2 = 4.0\n", + "\n", + " if CLIENT_VERSION >= (0, 5, 0):\n", + " logfile_path: str = solvers_api.get_job_output_logfile(\n", + " solver.id, solver.version, job.id\n", + " )\n", + " zip_path = Path(logfile_path)\n", + "\n", + " with TemporaryDirectory() as tmp_dir:\n", + " with ZipFile(f\"{zip_path}\") as fzip:\n", + " fzip.extractall(tmp_dir)\n", + " logfiles = list(Path(tmp_dir).glob(\"*.log*\"))\n", + " print(\"Unzipped\", logfiles[0], \"contains:\\n\", logfiles[0].read_text())\n", + " #\n", + " # Unzipped extracted/sleeper_2.0.2.logs contains:\n", + " # 2022-06-01T18:15:00.405035847+02:00 Entrypoint for stage production ...\n", + " # 2022-06-01T18:15:00.421279969+02:00 User : uid=0(root) gid=0(root) groups=0(root)\n", + " # 2022-06-01T18:15:00.421560331+02:00 Workdir : /home/scu\n", + " # ...\n", + " # 2022-06-01T18:15:00.864550043+02:00 \n", + " # 2022-06-01T18:15:03.923876794+02:00 Will sleep for 3 seconds\n", + " # 2022-06-01T18:15:03.924473521+02:00 [PROGRESS] 1/3...\n", + " # 2022-06-01T18:15:03.925021846+02:00 Remaining sleep time 0.9999995231628418\n", + " # 2022-06-01T18:15:03.925558026+02:00 [PROGRESS] 2/3...\n", + " # 2022-06-01T18:15:03.926103062+02:00 Remaining sleep time 0.9999985694885254\n", + " # 2022-06-01T18:15:03.926643184+02:00 [PROGRESS] 3/3...\n", + " # 2022-06-01T18:15:03.933544384+02:00 Remaining sleep time 0.9999983310699463\n", + "\n", + " download_path: str = files_api.download_file(file_id=outputs.results[\"output_1\"].id)\n", + " print(Path(download_path).read_text())\n", + " #\n", + " # 7\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c0092d84", + "metadata": {}, + "source": [ + "The script above\n", + "\n", + "1. Uploads a file ``file_with_number.txt``\n", + "2. Selects version ``2.0.2`` of the ``sleeper``\n", + "3. Runs the ``sleeper`` and provides a reference to the uploaded file and other values as input parameters\n", + "4. Monitors the status of the solver while it is running in the platform\n", + "5. When the execution completes, it checks the outputs\n", + "6. The logs are downloaded, unzipped and saved to a new ```extracted``` directory\n", + "7. One of the outputs is a file and it is downloaded\n", + "\n", + "\n", + "#### Files\n", + "\n", + "Files used as input to solvers or produced by solvers in the platform are accessible in the **files** section and specifically with the ``FilesApi`` class.\n", + "In order to use a file as input, it has to be uploaded first and the reference used in the corresponding solver's input." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d6d2dfdb", + "metadata": {}, + "source": [ + "```python\n", + "files_api = FilesApi(api_client)\n", + "input_file: File = files_api.upload_file(file=\"file_with_number.txt\")\n", + "\n", + "\n", + "# ...\n", + "\n", + "\n", + "outputs: JobOutputs = solvers_api.get_job_outputs(solver.id, solver.version, job.id)\n", + "results_file: File = outputs.results[\"output_1\"]\n", + "download_path: str = files_api.download_file(file_id=results_file.id)\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e40ad2a5", + "metadata": {}, + "source": [ + "In the snippet above, ``input_file`` is a ``File`` reference to the uploaded file and that is passed as input to the solver. Analogously, ``results_file`` is a ``File`` produced by the solver and that can also be downloaded.\n", + "\n", + "\n", + "#### Solvers, Inputs and Outputs\n", + "\n", + "The inputs and outputs are specific for every solver. Every input/output has a name and an associated type that can be as simple as booleans, numbers, strings ... or more complex as files. You can find this information in the UI under Services Tab, selecting the service card > Information > Raw metadata. For instance, the ``sleeper`` version ``2.0.2`` has the following ``raw-metadata``:" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f5262250", + "metadata": { + "attributes": { + "classes": [ + "json" + ], + "id": "" + } + }, + "source": [ + "```json\n", + "{\n", + " inputs: {\n", + " 'input_1': {'description': 'Pick a file containing only one '\n", + " 'integer',\n", + " 'displayOrder': 1,\n", + " 'fileToKeyMap': {'single_number.txt': 'input_1'},\n", + " 'label': 'File with int number',\n", + " 'type': 'data:text/plain'},\n", + " 'input_2': {'defaultValue': 2,\n", + " 'description': 'Choose an amount of time to sleep',\n", + " 'displayOrder': 2,\n", + " 'label': 'Sleep interval',\n", + " 'type': 'integer',\n", + " 'unit': 'second'},\n", + " 'input_3': {'defaultValue': False,\n", + " 'description': 'If set to true will cause service to '\n", + " 'fail after it sleeps',\n", + " 'displayOrder': 3,\n", + " 'label': 'Fail after sleep',\n", + " 'type': 'boolean'},\n", + " }\n", + "}\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1584bf07", + "metadata": {}, + "source": [ + "So, the inputs can be set as follows" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5337fcae", + "metadata": {}, + "source": [ + "```python\n", + "# ...\n", + "job = solvers_api.create_job(\n", + " solver.id,\n", + " solver.version,\n", + " job_inputs=JobInputs(\n", + " {\n", + " \"input_1\": uploaded_input_file,\n", + " \"input_2\": 3 * n, # sleep time in secs\n", + " \"input_3\": bool(n % 2), # fail after sleep?\n", + " }\n", + " ),\n", + " )\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "8dfa35ad", + "metadata": {}, + "source": [ + "And the metadata for the outputs are" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d4222e74", + "metadata": { + "attributes": { + "classes": [ + "json" + ], + "id": "" + } + }, + "source": [ + "```json\n", + "{\n", + " 'outputs': {'output_1': {'description': 'Integer is generated in range [1-9]',\n", + " 'displayOrder': 1,\n", + " 'fileToKeyMap': {'single_number.txt': 'output_1'},\n", + " 'label': 'File containing one random integer',\n", + " 'type': 'data:text/plain'},\n", + " 'output_2': {'description': 'Interval is generated in range '\n", + " '[1-9]',\n", + " 'displayOrder': 2,\n", + " 'label': 'Random sleep interval',\n", + " 'type': 'integer',\n", + " 'unit': 'second'}},\n", + "}\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "97e73630", + "metadata": {}, + "source": [ + "so this information determines which output corresponds to a number or a file in the following snippet" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "36e8f03b", + "metadata": {}, + "source": [ + "```python\n", + "# ...\n", + "\n", + "outputs: JobOutputs = solvers_api.get_job_outputs(solver.id, solver.version, job.id)\n", + "\n", + "output_file = outputs.results[\"output_1\"]\n", + "number = outputs.results[\"output_2\"]\n", + "\n", + "assert status.state == \"SUCCESS\"\n", + "\n", + "\n", + "assert isinstance(output_file, File)\n", + "assert isinstance(number, float)\n", + "\n", + "# output file exists\n", + "assert files_api.get_file(output_file.id) == output_file\n", + "\n", + "# can download and open\n", + "download_path: str = files_api.download_file(file_id=output_file.id)\n", + "assert float(Path(download_path).read_text()), \"contains a random number\"\n", + "\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a58035b0", + "metadata": {}, + "source": [ + "#### Job Status\n", + "\n", + "Once the client script triggers the solver, the solver runs in the platform and the script is freed. Sometimes, it is convenient to monitor the status of the run to see e.g. the progress of the execution or if the run was completed.\n", + "\n", + "A solver runs in a plaforma starts a ``Job``. Using the ``solvers_api``, allows us to inspect the ``Job`` and get a ``JobStatus`` with information about its status. For instance" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "93817d1e", + "metadata": {}, + "source": [ + "```python \n", + " status: JobStatus = solvers_api.start_job(solver.id, solver.version, job.id)\n", + " while not status.stopped_at:\n", + " time.sleep(3)\n", + " status = solvers_api.inspect_job(solver.id, solver.version, job.id)\n", + " print(\"Solver progress\", f\"{status.progress}/100\", flush=True)\n", + "``` " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5180b589", + "metadata": {}, + "source": [ + "#### Logs\n", + "\n", + "When a solver runs, it will generate logs during execution which are then saved as .log files. Starting from the osparc Python Client version 0.5.0, The ``solvers_api`` also allows us to obtain the ``logfile_path`` associated with a particular ``Job``. This is a zip file that can then be extracted and saved. For instance" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "417d4663", + "metadata": {}, + "source": [ + "```python\n", + "logfile_path: str = solvers_api.get_job_output_logfile(\n", + " solver.id, solver.version, job.id\n", + ")\n", + "zip_path = Path(logfile_path)\n", + "\n", + "extract_dir = Path(\"./extracted\")\n", + "extract_dir.mkdir()\n", + "\n", + "with ZipFile(f\"{zip_path}\") as fzip:\n", + " fzip.extractall(f\"{extract_dir}\")\n", + "```" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "72d60050", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "- [osparc API python client] documentation\n", + "- [osparc API] documentation\n", + "- A full script with this tutorial: [``sleeper.py``](https://github.com/ITISFoundation/osparc-simcore/blob/master/tests/public-api/examples/sleeper.py)\n", + "\n", + "[osparc API python client]:https://itisfoundation.github.io/osparc-simcore-clients/#/\n", + "[osparc API]:https://api.osparc.io/dev/doc#/" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "", + "language": "python", + "name": "" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/clients/python/docs/v0.5.0/README.md b/clients/python/docs/v0.5.0/README.md index 7d58fd16..e02cdc9c 100644 --- a/clients/python/docs/v0.5.0/README.md +++ b/clients/python/docs/v0.5.0/README.md @@ -5,4 +5,4 @@ ## Tutorials -- [Basic tutorial](clients/python/artifacts/docs/BasicTutorial.md) ([Download as BasicTutorial.ipynb](clients/python/docs/BasicTutorial.ipynb ":ignore title")) +- [Basic tutorial](clients/python/artifacts/docs/BasicTutorial_v0.5.0.md) ([Download as BasicTutorial.ipynb](clients/python/docs/BasicTutorial_v0.5.0.ipynb ":ignore title")) diff --git a/clients/python/docs/v0.6.0/README.md b/clients/python/docs/v0.6.0/README.md new file mode 100644 index 00000000..a81e2dea --- /dev/null +++ b/clients/python/docs/v0.6.0/README.md @@ -0,0 +1,8 @@ +# OSPARC Python package + +- API version: 0.5.0 +- Package version: 0.6.0 + +## Tutorials + +- [Basic tutorial](clients/python/artifacts/docs/BasicTutorial_v0.6.0.md) ([Download as BasicTutorial.ipynb](clients/python/docs/BasicTutorial_v0.6.0.ipynb ":ignore title")) diff --git a/clients/python/requirements/test.txt b/clients/python/requirements/test.txt index 3f193631..fd65fc23 100644 --- a/clients/python/requirements/test.txt +++ b/clients/python/requirements/test.txt @@ -13,3 +13,4 @@ pytest pytest-env pytest-html typer +packaging diff --git a/clients/python/test/e2e/_utils.py b/clients/python/test/e2e/_utils.py new file mode 100644 index 00000000..7a6c68eb --- /dev/null +++ b/clients/python/test/e2e/_utils.py @@ -0,0 +1,30 @@ +import json +import os +from pathlib import Path + +import osparc +import pytest +from packaging.version import Version + +_config: Path = ( + Path(__file__).parent.parent.parent.parent.parent / "api" / "config.json" +) +assert _config.is_file() + + +def osparc_dev_features_enabled() -> bool: + return os.environ.get("OSPARC_DEV_FEATURES_ENABLED") == "1" + + +def requires_dev_features(test): + repo_version: Version = Version( + json.loads(_config.read_text())["python"]["version"] + ) + if Version(osparc.__version__) < repo_version or not osparc_dev_features_enabled(): + return pytest.mark.skip( + ( + f"{osparc.__version__=}<{str(repo_version)} " + "or {osparc_dev_features_enabled()=}" + ) + )(test) + return test diff --git a/clients/python/test/e2e/ci/_data_classes.py b/clients/python/test/e2e/ci/_data_classes.py index 62a7dddb..73a7eef3 100644 --- a/clients/python/test/e2e/ci/_data_classes.py +++ b/clients/python/test/e2e/ci/_data_classes.py @@ -13,21 +13,21 @@ class ServerConfig(BaseModel): """Holds data about server configuration""" - osparc_api_host: str - osparc_api_key: str - osparc_api_secret: str + api_host: str + api_key: str + api_secret: str @property def url(self) -> ParseResult: - return urlparse(f"{self.osparc_api_host}") + return urlparse(f"{self.api_host}") @property def key(self) -> str: - return self.osparc_api_key + return self.api_key @property def secret(self) -> str: - return self.osparc_api_secret + return self.api_secret def is_empty(v): @@ -39,13 +39,14 @@ class ClientConfig(BaseModel): This data should uniquely determine how to install client """ - osparc_client_version: Optional[str] = None - osparc_client_repo: Optional[str] = None - osparc_client_branch: Optional[str] = None - osparc_client_workflow: Optional[str] = None - osparc_client_runid: Optional[str] = None + client_version: Optional[str] = None + client_repo: Optional[str] = None + client_branch: Optional[str] = None + client_dev_features: bool = False + client_workflow: Optional[str] = None + client_runid: Optional[str] = None - @field_validator("osparc_client_version") + @field_validator("client_version") def validate_client(cls, v): if (not is_empty(v)) and (not v == "latest"): try: @@ -57,54 +58,51 @@ def validate_client(cls, v): @model_validator(mode="after") def check_consistency(self) -> "ClientConfig": msg: str = ( - f"Recieved osparc_client_version={self.osparc_client_version}, " - f"osparc_client_repo={self.osparc_client_repo}" - "and osparc_client_branch={self.osparc_client_branch}. " + f"Recieved client_version={self.client_version}, " + f"client_repo={self.client_repo}" + "and client_branch={self.client_branch}. " "Either a version or a repo, branch pair must be specified. Not both." ) # check at least one is empty if not ( - is_empty(self.osparc_client_version) - or ( - is_empty(self.osparc_client_repo) - and is_empty(self.osparc_client_branch) - ) + is_empty(self.client_version) + or (is_empty(self.client_repo) and is_empty(self.client_branch)) ): raise ValueError(msg) # check not both empty - if is_empty(self.osparc_client_version) and ( - is_empty(self.osparc_client_repo) and is_empty(self.osparc_client_branch) + if is_empty(self.client_version) and ( + is_empty(self.client_repo) and is_empty(self.client_branch) ): raise ValueError(msg) - if is_empty(self.osparc_client_version): + if is_empty(self.client_version): if ( - is_empty(self.osparc_client_repo) - or is_empty(self.osparc_client_branch) - or is_empty(self.osparc_client_workflow) - or is_empty(self.osparc_client_runid) + is_empty(self.client_repo) + or is_empty(self.client_branch) + or is_empty(self.client_workflow) + or is_empty(self.client_runid) ): raise ValueError(msg) return self @property def version(self) -> Optional[str]: - return self.osparc_client_version + return self.client_version @property def repo(self) -> Optional[str]: - return self.osparc_client_repo + return self.client_repo @property def branch(self) -> Optional[str]: - return self.osparc_client_branch + return self.client_branch @property def workflow(self) -> Optional[str]: - return self.osparc_client_workflow + return self.client_workflow @property def runid(self) -> Optional[str]: - return self.osparc_client_runid + return self.client_runid @property def compatibility_ref(self) -> str: @@ -113,7 +111,10 @@ def compatibility_ref(self) -> str: return "production" else: assert isinstance(self.branch, str) - return self.branch + if self.client_dev_features: + return f"{self.branch}+dev_features" + else: + return f"{self.branch}-dev_features" @property def client_ref(self) -> str: @@ -123,7 +124,9 @@ def client_ref(self) -> str: return self.version else: assert isinstance(self.branch, str) - return self.branch + if self.client_dev_features: + return f"{self.branch}+dev_features" + return f"{self.branch}-dev_features" class PytestConfig(BaseModel): diff --git a/clients/python/test/e2e/ci/generate_pytest_ini.py b/clients/python/test/e2e/ci/generate_pytest_ini.py index e4ab8db3..39a4a2de 100644 --- a/clients/python/test/e2e/ci/generate_pytest_ini.py +++ b/clients/python/test/e2e/ci/generate_pytest_ini.py @@ -59,6 +59,9 @@ def main(client_config: str, server_config: str) -> None: envs.append(f"OSPARC_API_HOST={server_cfg.url.geturl()}") envs.append(f"OSPARC_API_KEY={server_cfg.key}") envs.append(f"OSPARC_API_SECRET={server_cfg.secret}") + envs.append( + f"OSPARC_DEV_FEATURES_ENABLED=" f"{1 if client_cfg.client_dev_features else 0}" + ) html_log: Path = Path("../../") / ( _ARTIFACTS_DIR diff --git a/clients/python/test/e2e/ci/install_osparc_python_client.bash b/clients/python/test/e2e/ci/install_osparc_python_client.bash index 46d35414..354d7e53 100644 --- a/clients/python/test/e2e/ci/install_osparc_python_client.bash +++ b/clients/python/test/e2e/ci/install_osparc_python_client.bash @@ -12,25 +12,25 @@ doc+="\tA single json string. This json string is expected to be the output of s print_doc() { echo -e "$doc"; } [ $# -eq 0 ] && print_doc && exit 0 -osparc_client_config=$1 +client_config=$1 -if [[ $(echo "$osparc_client_config" | jq 'has("osparc_client_version")') == "true" ]]; then - osparc_client_version=$(echo "${osparc_client_config}" | jq -r .osparc_client_version) +if [[ $(echo "$client_config" | jq 'has("client_version")') == "true" ]]; then + client_version=$(echo "${client_config}" | jq -r .client_version) v_string="" - if [[ "${osparc_client_version}" != "latest" ]]; then - v_string="==${osparc_client_version}" + if [[ "${client_version}" != "latest" ]]; then + v_string="==${client_version}" fi python -m pip install osparc"${v_string}" --force-reinstall else - osparc_client_repo=$(echo "${osparc_client_config}" | jq -r .osparc_client_repo) - osparc_client_runid=$(echo "${osparc_client_config}" | jq -r .osparc_client_runid) + client_repo=$(echo "${client_config}" | jq -r .client_repo) + client_runid=$(echo "${client_config}" | jq -r .client_runid) tmp_dir=$(mktemp -d) pushd "${tmp_dir}" - echo "gh run download ${osparc_client_runid} --repo=${osparc_client_repo}" - gh run download "${osparc_client_runid}" --repo="${osparc_client_repo}" + echo "gh run download ${client_runid} --repo=${client_repo}" + gh run download "${client_runid}" --repo="${client_repo}" popd osparc_wheel=$(ls "${tmp_dir}"/osparc_python_wheels/osparc-*.whl) python -m pip install "${osparc_wheel}" --find-links="${tmp_dir}"/osparc_python_wheels --force-reinstall diff --git a/clients/python/test/e2e/ci/preprocess_client_config.bash b/clients/python/test/e2e/ci/preprocess_client_config.bash index dfd8761f..b6af31a1 100644 --- a/clients/python/test/e2e/ci/preprocess_client_config.bash +++ b/clients/python/test/e2e/ci/preprocess_client_config.bash @@ -7,10 +7,10 @@ set -o pipefail # don't hide errors within pipes doc="Setup client configuration for e2e testing of the osparc python client\n" doc+="Input:\n" doc+="------\n" -doc+="\tA single json string containing either the field osparc_client_version or the two fields osparc_client_repo and osparc_client_branch.\n" -doc+="\tosparc_client_version can either be the version of a client (e.g. \"0.5.0\") or \"latest\"\n" -doc+="\tExample 1: bash setup_client_config.bash '{\"osparc_client_repo\": \"ITISFoundation/osparc-simcore-clients\", \"osparc_client_branch\": \"master\"}'\n" -doc+="\tExample 2: bash setup_client_config.bash '{\"osparc_client_version\": \"0.5.0\"}'\n" +doc+="\tA single json string containing either the field client_version or the two fields client_repo and osparc_client_branch with an optional field client_dev_features.\n" +doc+="\tclient_version can either be the version of a client (e.g. \"0.5.0\") or \"latest\"\n" +doc+="\tExample 1: bash setup_client_config.bash '{\"client_repo\": \"ITISFoundation/osparc-simcore-clients\", \"client_branch\": \"master\"}'\n" +doc+="\tExample 2: bash setup_client_config.bash '{\"client_version\": \"0.5.0\"}'\n" doc+="Output:\n" doc+="-------\n" doc+="\tA json string which, when passed to install_osparc_python_client.bash installs the wanted python client" @@ -18,29 +18,29 @@ doc+="\tA json string which, when passed to install_osparc_python_client.bash in print_doc() { echo -e "$doc"; } [ $# -eq 0 ] && print_doc && exit 0 -osparc_client_workflow=publish-and-test-python-client -osparc_client_config=$1 +client_workflow=publish-and-test-python-client +client_config=$1 # extract keys from input json -if [[ $(echo "$osparc_client_config" | jq 'has("osparc_client_version")') == "true" ]]; then - if [[ $(echo "$osparc_client_config" | jq 'length') != 1 ]]; then - echo "${osparc_client_config} was invalid" +if [[ $(echo "$client_config" | jq 'has("client_version")') == "true" ]]; then + if [[ $(echo "$client_config" | jq 'length') != 1 ]]; then + echo "${client_config} was invalid" exit 1 fi else - if [[ $(echo "$osparc_client_config" | jq 'length') != 2 ]]; then - echo "${osparc_client_config} was invalid" + if [[ $(echo "$client_config" | jq 'length') != 2 && $(echo "$client_config" | jq 'length') != 3 ]]; then + echo "${client_config} was invalid" exit 1 fi - if [[ $(echo "$osparc_client_config" | jq 'has("osparc_client_branch")') != "true" || $(echo "$osparc_client_config" | jq 'has("osparc_client_repo")') != "true" ]]; then - echo "${osparc_client_config} was invalid" + if [[ $(echo "$client_config" | jq 'has("client_branch")') != "true" || $(echo "$client_config" | jq 'has("client_repo")') != "true" ]]; then + echo "${client_config} was invalid" exit 1 fi - osparc_client_repo=$(echo "$osparc_client_config" | jq -r '.osparc_client_repo') - osparc_client_branch=$(echo "$osparc_client_config" | jq -r '.osparc_client_branch') - osparc_client_runid=$(gh run list --repo="${osparc_client_repo}" --branch="${osparc_client_branch}" --workflow="${osparc_client_workflow}" --limit=100 --json=databaseId,status --jq='map(select(.status=="completed")) | .[0].databaseId') - osparc_client_config=$(echo "${osparc_client_config}" | jq --arg cwfw "${osparc_client_workflow}" '. += {"osparc_client_workflow": $cwfw}') - osparc_client_config=$(echo "${osparc_client_config}" | jq --arg crid "${osparc_client_runid}" '. += {"osparc_client_runid": $crid}') + client_repo=$(echo "$client_config" | jq -r '.client_repo') + client_branch=$(echo "$client_config" | jq -r '.client_branch') + client_runid=$(gh run list --repo="${client_repo}" --branch="${client_branch}" --workflow="${client_workflow}" --limit=100 --json=databaseId,status --jq='map(select(.status=="completed")) | .[0].databaseId') + client_config=$(echo "${client_config}" | jq --arg cwfw "${client_workflow}" '. += {"client_workflow": $cwfw}') + client_config=$(echo "${client_config}" | jq --arg crid "${client_runid}" '. += {"client_runid": $crid}') fi -echo "${osparc_client_config}" +echo "${client_config}" diff --git a/clients/python/test/e2e/ci/run_e2e_python_tests.bash b/clients/python/test/e2e/ci/run_e2e_python_tests.bash index e40a9c14..65e1d8e4 100644 --- a/clients/python/test/e2e/ci/run_e2e_python_tests.bash +++ b/clients/python/test/e2e/ci/run_e2e_python_tests.bash @@ -10,11 +10,6 @@ doc+="Input:\n" doc+="------\n" doc+="\tTwo json strings: A client json configuration and a server json configuration. Example: \"bash run_e2e_python_tests.bash -c -s \".\n" doc+="\tThe client array must adhere to the requirements of setup_client_config.bash (run \"bash setup_client_config.bash\").\n" -doc+="\tThe server json string must be an array of json objects. Each json object must contain the following fields:\n" -doc+="\t\t - OSPARC_API_HOST\n" -doc+="\t\t - OSPARC_API_KEY\n" -doc+="\t\t - OSPARC_API_SECRET\n" -doc+="\tExample: [{\"OSPARC_API_HOST\":\"https://api.osparc-master.speag.com\", \"OSPARC_API_KEY\":\"mykey\", \"OSPARC_API_SECRET\":\"mysecret\"}] \n" doc+="Output:\n" doc+="-------\n" doc+="\tTest results are stored in clients/python/artifacts/e2e/.json and the \"pyproject.toml\" file from which one can completely\n" diff --git a/clients/python/test/e2e/data/server_client_compatibility.json b/clients/python/test/e2e/data/server_client_compatibility.json index 2ab710d3..06319f27 100644 --- a/clients/python/test/e2e/data/server_client_compatibility.json +++ b/clients/python/test/e2e/data/server_client_compatibility.json @@ -1,50 +1,75 @@ { "0": { "server": "api.osparc-master.speag.com", - "client": "master", + "client": "master+dev_features", "is_compatible": true }, "1": { "server": "api.osparc-master.speag.com", - "client": "production", + "client": "master-dev_features", "is_compatible": true }, "2": { + "server": "api.osparc-master.speag.com", + "client": "production", + "is_compatible": true + }, + "3": { "server": "api.osparc-staging.speag.com", - "client": "master", + "client": "master+dev_features", "is_compatible": false }, - "3": { + "4": { + "server": "api.osparc-staging.speag.com", + "client": "master-dev_features", + "is_compatible": true + }, + "5": { "server": "api.osparc-staging.speag.com", "client": "production", "is_compatible": true }, - "4": { + "6": { "server": "api.osparc.speag.com", - "client": "master", + "client": "master+dev_features", "is_compatible": false }, - "5": { + "7": { + "server": "api.osparc.speag.com", + "client": "master-dev_features", + "is_compatible": false + }, + "8": { "server": "api.osparc.speag.com", "client": "production", "is_compatible": true }, - "6": { + "9": { "server": "api.staging.osparc.io", - "client": "master", + "client": "master+dev_features", "is_compatible": false }, - "7": { + "10": { + "server": "api.staging.osparc.io", + "client": "master-dev_features", + "is_compatible": true + }, + "11": { "server": "api.staging.osparc.io", "client": "production", "is_compatible": true }, - "8": { + "12": { "server": "api.osparc.io", - "client": "master", + "client": "master+dev_features", "is_compatible": false }, - "9": { + "13": { + "server": "api.osparc.io", + "client": "master-dev_features", + "is_compatible": false + }, + "14": { "server": "api.osparc.io", "client": "production", "is_compatible": true diff --git a/clients/python/test/e2e/test_files_api.py b/clients/python/test/e2e/test_files_api.py index fecbf5fa..e221ae73 100644 --- a/clients/python/test/e2e/test_files_api.py +++ b/clients/python/test/e2e/test_files_api.py @@ -3,8 +3,8 @@ import osparc import pytest +from _utils import requires_dev_features from conftest import _KB -from packaging.version import Version def _hash_file(file: Path) -> str: @@ -19,10 +19,7 @@ def _hash_file(file: Path) -> str: return sha256.hexdigest() -@pytest.mark.skipif( - Version(osparc.__version__) < Version("0.6.0"), - reason=f"osparc.__version__={osparc.__version__} is older than 0.6.0", -) +@requires_dev_features def test_upload_file(tmp_file: Path, cfg: osparc.Configuration) -> None: """Test that we can upload a file via the multipart upload""" tmp_path: Path = tmp_file.parent @@ -41,10 +38,7 @@ def test_upload_file(tmp_file: Path, cfg: osparc.Configuration) -> None: files_api.delete_file(uploaded_file1.id) -@pytest.mark.skipif( - Version(osparc.__version__) < Version("0.6.0"), - reason=f"osparc.__version__={osparc.__version__} is older than 0.6.0", -) +@requires_dev_features @pytest.mark.parametrize("use_checksum", [True, False]) @pytest.mark.parametrize("use_id", [True, False]) def test_search_files( diff --git a/clients/python/test/e2e/test_notebooks.py b/clients/python/test/e2e/test_notebooks.py index f976d741..d402e353 100644 --- a/clients/python/test/e2e/test_notebooks.py +++ b/clients/python/test/e2e/test_notebooks.py @@ -1,14 +1,19 @@ import shutil import sys from pathlib import Path -from typing import Any, List +from typing import Any, Dict, List import osparc import papermill as pm import pytest +from packaging.version import Version docs_dir: Path = Path(__file__).parent.parent.parent / "docs" all_notebooks: List[Path] = list(docs_dir.rglob("*.ipynb")) +min_version_reqs: Dict[str, Version] = { + "BasicTutorial_v0.5.0.ipynb": Version("0.5.0"), + "BasicTutorial_v0.6.0.ipynb": Version("0.6.0"), +} def test_notebook_config(tmp_path: Path): @@ -25,15 +30,27 @@ def test_notebook_config(tmp_path: Path): }, ) assert len(all_notebooks) > 0, f"Did not find any notebooks in {docs_dir}" + min_keys: set = set(min_version_reqs.keys()) + notebook_names: set = set(pth.name for pth in all_notebooks) + msg: str = ( + f"Must specify max version for: {notebook_names-min_keys}." + f" The following keys can be deleted: {min_keys - notebook_names}" + ) + assert min_keys == notebook_names, msg -@pytest.mark.parametrize("notebook", all_notebooks) +@pytest.mark.parametrize("notebook", all_notebooks, ids=lambda nb: nb.name) def test_run_notebooks(tmp_path: Path, notebook: Path, params: dict[str, Any] = {}): """Run all notebooks in the documentation""" - print(f"Running {notebook.name} with parameters {params}") assert ( notebook.is_file() ), f"{notebook.name} is not a file (full path: {notebook.resolve()})" + if min_version := min_version_reqs.get(notebook.name): + if Version(osparc.__version__) < min_version: + pytest.skip( + f"Skipping {notebook.name} because " + f"{osparc.__version__=} < {min_version=}" + ) tmp_nb = tmp_path / notebook.name shutil.copy(notebook, tmp_nb) assert tmp_nb.is_file(), "Did not succeed in copying notebook" diff --git a/clients/python/test/e2e/test_solvers_api.py b/clients/python/test/e2e/test_solvers_api.py index 919b8404..43214553 100644 --- a/clients/python/test/e2e/test_solvers_api.py +++ b/clients/python/test/e2e/test_solvers_api.py @@ -1,12 +1,8 @@ import osparc -import pytest -from packaging.version import Version +from _utils import requires_dev_features -@pytest.mark.skipif( - Version(osparc.__version__) < Version("0.6.0"), - reason=f"osparc.__version__={osparc.__version__} is older than 0.6.0", -) +@requires_dev_features def test_jobs(cfg: osparc.Configuration): """Test the jobs method diff --git a/docs/navbar.md b/docs/navbar.md index c234c0d3..af1b7640 100644 --- a/docs/navbar.md +++ b/docs/navbar.md @@ -1,3 +1,4 @@ * Python * [v0.5.0](clients/python/docs/v0.5.0/README.md) + * [v0.6.0](clients/python/docs/v0.6.0/README.md)