From 64f28fd6666e156089dbf59a457a41e3fdffdfbb Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Tue, 1 Oct 2024 00:21:31 +0300 Subject: [PATCH] Move to alpine and improve security & smoke test --- .github/workflows/core-test.yml | 18 +++- .github/workflows/detect-changes-matrix.yml | 1 + Makefile | 7 +- integrations/_infra/Dockerfile | 72 +++++++++++--- integrations/_infra/init.sh | 5 - port_ocean/log/sensetive.py | 2 +- port_ocean/tests/helpers/fixtures.py | 104 +++----------------- port_ocean/tests/helpers/integration.py | 31 ++++++ port_ocean/tests/helpers/port_client.py | 21 ++++ port_ocean/tests/helpers/smoke_test.py | 82 +++++++++++++++ port_ocean/tests/test_smoke.py | 2 +- pyproject.toml | 5 +- scripts/clean-smoke-test.py | 10 ++ scripts/run-smoke-test.sh | 4 +- 14 files changed, 244 insertions(+), 120 deletions(-) delete mode 100644 integrations/_infra/init.sh create mode 100644 port_ocean/tests/helpers/integration.py create mode 100644 port_ocean/tests/helpers/port_client.py create mode 100644 port_ocean/tests/helpers/smoke_test.py create mode 100755 scripts/clean-smoke-test.py diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-test.yml index ac4c981b10..46f53bd63d 100644 --- a/.github/workflows/core-test.yml +++ b/.github/workflows/core-test.yml @@ -29,16 +29,16 @@ jobs: run: | make install - - name: Build core for smoke test - run: | - make build - - name: Unit Test Core env: PYTEST_ADDOPTS: --junitxml=junit/unit-test-results-ocean/core.xml run: | make test + - name: Build core for smoke test + run: | + make build + - name: Run fake integration for core test env: PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} @@ -58,6 +58,16 @@ jobs: run: | make test/smoke + - name: Cleanup Smoke Test + env: + PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml + PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} + PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} + PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} + SMOKE_TEST_SUFFIX: ${{ github.run_id }} + run: | + make test/smoke + - name: Install current core for all integrations run: | echo "Installing local core for all integrations" diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml index facb2cecd0..91dd9c31fc 100644 --- a/.github/workflows/detect-changes-matrix.yml +++ b/.github/workflows/detect-changes-matrix.yml @@ -41,6 +41,7 @@ jobs: integrations: - 'integrations/**' - '!integrations/**/*.md' + - '!integrations/_infra/*' - name: Set integrations and all matrix id: set-all-matrix diff --git a/Makefile b/Makefile index 0d3be2993a..b9d3be75d6 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ define deactivate_virtualenv fi endef -.SILENT: install install/all test/all lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all +.SILENT: install install/all test/all test/smoke clean/smoke lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all # Install dependencies @@ -115,11 +115,14 @@ new: $(ACTIVATE) && poetry run ocean new ./integrations --public test: - $(ACTIVATE) && pytest + $(ACTIVATE) && pytest -m 'not smoke' test/smoke: $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} pytest -m smoke +clean/smoke: + $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./scripts/clean-smoke-test.py + test/watch: $(ACTIVATE) && \ pytest \ diff --git a/integrations/_infra/Dockerfile b/integrations/_infra/Dockerfile index 23e7e3bff0..ab61fe4833 100644 --- a/integrations/_infra/Dockerfile +++ b/integrations/_infra/Dockerfile @@ -1,28 +1,72 @@ -FROM python:3.11-slim-bookworm +FROM python:3.11-alpine AS base ARG BUILD_CONTEXT + +ENV LIBRDKAFKA_VERSION=1.9.2 + +# Install system dependencies and libraries +RUN apk add --no-cache \ + gcc \ + musl-dev \ + build-base \ + bash \ + oniguruma-dev \ + make \ + autoconf \ + automake \ + libtool \ + curl \ + # librdkafka-dev \ + libffi-dev \ + # Install community librdkafka-dev since the default in alpine is older + && echo "@edge http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \ + && echo "@edgecommunity http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && apk add --no-cache alpine-sdk "librdkafka@edgecommunity>=${LIBRDKAFKA_VERSION}" "librdkafka-dev@edgecommunity>=${LIBRDKAFKA_VERSION}" \ + && curl -sSL https://install.python-poetry.org | python3 - \ + && /root/.local/bin/poetry config virtualenvs.in-project true + + +WORKDIR /app + +COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/ + +RUN /root/.local/bin/poetry install --without dev --no-root --no-interaction --no-ansi --no-cache && pip cache purge + +FROM python:3.11-alpine AS prod + ARG INTEGRATION_VERSION +ARG BUILD_CONTEXT LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION} # Used to ensure that new integrations will be public, see https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility -LABEL org.opencontainers.image.source https://github.com/port-labs/ocean +LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean -ENV LIBRDKAFKA_VERSION 1.9.2 +# Install only runtime dependencies +RUN apk add --no-cache \ + librdkafka-dev \ + bash \ + oniguruma-dev \ + # Install community librdkafka-dev since the default in alpine is older + && echo "@edge http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \ + && echo "@edgecommunity http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && apk add --no-cache alpine-sdk "librdkafka@edgecommunity>=${LIBRDKAFKA_VERSION}" "librdkafka-dev@edgecommunity>=${LIBRDKAFKA_VERSION}" \ + && test -e /usr/local/share/ca-certificates/cert.crt && update-ca-certificates || true WORKDIR /app -RUN apt update && \ - apt install -y wget make g++ libssl-dev autoconf automake libtool curl librdkafka-dev && \ - apt-get clean - -COPY ./integrations/_infra/init.sh /app/init.sh - -RUN chmod +x /app/init.sh +# Copy dependencies from the build stage +COPY --from=base /app /app +# Copy the application code COPY ./${BUILD_CONTEXT} /app -COPY ./integrations/_infra/Makefile /app/Makefile - -RUN export POETRY_VIRTUALENVS_CREATE=false && make install/prod && pip cache purge +# Ensure that ocean is available for all in path +RUN chmod a+x /app/.venv/bin/ocean \ + && ln -s /app/.venv/bin/ocean /usr/bin/ocean \ + # # Fix security issues + && apk upgrade busybox --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main \ + # Clean up old setuptools + && pip uninstall -y setuptools py3-setuptools -ENTRYPOINT ./init.sh +# Run the application +CMD ["ocean", "sail"] diff --git a/integrations/_infra/init.sh b/integrations/_infra/init.sh deleted file mode 100644 index 701f61ae1c..0000000000 --- a/integrations/_infra/init.sh +++ /dev/null @@ -1,5 +0,0 @@ -if test -e /usr/local/share/ca-certificates/cert.crt; then - update-ca-certificates -fi - -ocean sail \ No newline at end of file diff --git a/port_ocean/log/sensetive.py b/port_ocean/log/sensetive.py index e4779cfcc4..c5d6b723ca 100644 --- a/port_ocean/log/sensetive.py +++ b/port_ocean/log/sensetive.py @@ -21,7 +21,7 @@ "GitHub": r"[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]", "Google Cloud Platform API Key": r"AIza[0-9A-Za-z\\-_]{35}", "Google Cloud Platform OAuth": r"[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com", - "Google (GCP) Service-account": r'"type": "service_account"', + "Google (GCP) Service-account": f'"type":{" "}"service_account"', "Google OAuth Access Token": r"ya29\\.[0-9A-Za-z\\-_]+", "Connection String": r"[a-zA-Z]+:\/\/[^/\s]+:[^/\s]+@[^/\s]+\/[^/\s]+", } diff --git a/port_ocean/tests/helpers/fixtures.py b/port_ocean/tests/helpers/fixtures.py index 200c4ea195..7c716afb96 100644 --- a/port_ocean/tests/helpers/fixtures.py +++ b/port_ocean/tests/helpers/fixtures.py @@ -1,9 +1,8 @@ -from os import environ, path -from typing import Any, AsyncGenerator, Callable, List, Tuple, Union +from os import path +from typing import Any, Callable, List, Tuple +import pytest import pytest_asyncio -from loguru import logger -from pydantic import BaseModel from port_ocean.clients.port.client import PortClient from port_ocean.core.handlers.port_app_config.models import ResourceConfig @@ -12,96 +11,19 @@ get_integation_resource_configs, get_integration_ocean_app, ) +from port_ocean.tests.helpers.smoke_test import ( + SmokeTestDetails, + get_port_client_for_fake_integration, + get_smoke_test_details, +) -def get_port_client_for_integration( - client_id: str, - client_secret: str, - integration_identifier: str, - integration_type: str, - integration_version: str, - base_url: Union[str, None], -) -> PortClient: - return PortClient( - base_url=base_url or "https://api.getport/io", - client_id=client_id, - client_secret=client_secret, - integration_identifier=integration_identifier, - integration_type=integration_type, - integration_version=integration_version, - ) - - -async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None: - for blueprint in blueprints: - try: - bp = await client.get_blueprint(blueprint) - if bp is not None: - migration_id = await client.delete_blueprint( - identifier=blueprint, delete_entities=True - ) - if migration_id: - await client.wait_for_migration_to_complete( - migration_id=migration_id - ) - except Exception as bp_e: - logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}") - headers = await client.auth.headers() - try: - await client.client.delete( - f"{client.auth.api_url}/integrations/{client.integration_identifier}", - headers=headers, - ) - except Exception as int_e: - logger.info( - f"Failed to delete integration ({client.integration_identifier}): {int_e}" - ) - - -class SmokeTestDetails(BaseModel): - integration_identifier: str - blueprint_department: str - blueprint_person: str - - -@pytest_asyncio.fixture() -async def port_client_for_fake_integration() -> ( - AsyncGenerator[Tuple[SmokeTestDetails, PortClient], None] -): - blueprint_department = "fake-department" - blueprint_person = "fake-person" - integration_identifier = "smoke-test-integration" - smoke_test_suffix = environ.get("SMOKE_TEST_SUFFIX") - client_id = environ.get("PORT_CLIENT_ID") - client_secret = environ.get("PORT_CLIENT_SECRET") - - if not client_secret or not client_id: - assert False, "Missing port credentials" - - base_url = environ.get("PORT_BASE_URL") - integration_version = "0.1.1-dev" - integration_type = "smoke-test" - if smoke_test_suffix is not None: - integration_identifier = f"{integration_identifier}-{smoke_test_suffix}" - blueprint_person = f"{blueprint_person}-{smoke_test_suffix}" - blueprint_department = f"{blueprint_department}-{smoke_test_suffix}" - - client = get_port_client_for_integration( - client_id, - client_secret, - integration_identifier, - integration_type, - integration_version, - base_url, - ) +@pytest.fixture +def port_client_for_fake_integration() -> Tuple[SmokeTestDetails, PortClient]: + smoke_test_details = get_smoke_test_details() + port_client = get_port_client_for_fake_integration() - smoke_test_details = SmokeTestDetails( - integration_identifier=integration_identifier, - blueprint_person=blueprint_person, - blueprint_department=blueprint_department, - ) - yield smoke_test_details, client - await cleanup_integration(client, [blueprint_department, blueprint_person]) + return smoke_test_details, port_client @pytest_asyncio.fixture diff --git a/port_ocean/tests/helpers/integration.py b/port_ocean/tests/helpers/integration.py new file mode 100644 index 0000000000..18d63364b0 --- /dev/null +++ b/port_ocean/tests/helpers/integration.py @@ -0,0 +1,31 @@ +from typing import List + +from loguru import logger + +from port_ocean.clients.port.client import PortClient + + +async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None: + for blueprint in blueprints: + try: + bp = await client.get_blueprint(blueprint) + if bp is not None: + migration_id = await client.delete_blueprint( + identifier=blueprint, delete_entities=True + ) + if migration_id: + await client.wait_for_migration_to_complete( + migration_id=migration_id + ) + except Exception as bp_e: + logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}") + headers = await client.auth.headers() + try: + await client.client.delete( + f"{client.auth.api_url}/integrations/{client.integration_identifier}", + headers=headers, + ) + except Exception as int_e: + logger.info( + f"Failed to delete integration ({client.integration_identifier}): {int_e}" + ) diff --git a/port_ocean/tests/helpers/port_client.py b/port_ocean/tests/helpers/port_client.py new file mode 100644 index 0000000000..bbc9ff9503 --- /dev/null +++ b/port_ocean/tests/helpers/port_client.py @@ -0,0 +1,21 @@ +from typing import Union + +from port_ocean.clients.port.client import PortClient + + +def get_port_client_for_integration( + client_id: str, + client_secret: str, + integration_identifier: str, + integration_type: str, + integration_version: str, + base_url: Union[str, None], +) -> PortClient: + return PortClient( + base_url=base_url or "https://api.getport/io", + client_id=client_id, + client_secret=client_secret, + integration_identifier=integration_identifier, + integration_type=integration_type, + integration_version=integration_version, + ) diff --git a/port_ocean/tests/helpers/smoke_test.py b/port_ocean/tests/helpers/smoke_test.py new file mode 100644 index 0000000000..29c8886532 --- /dev/null +++ b/port_ocean/tests/helpers/smoke_test.py @@ -0,0 +1,82 @@ +from os import environ +from port_ocean.clients.port.client import PortClient + +from loguru import logger +from pydantic import BaseModel + +from port_ocean.tests.helpers.integration import cleanup_integration +from port_ocean.tests.helpers.port_client import get_port_client_for_integration + + +class SmokeTestDetails(BaseModel): + integration_identifier: str + blueprint_department: str + blueprint_person: str + integration_type: str + integration_version: str + + +def get_smoke_test_details() -> SmokeTestDetails: + blueprint_department = "fake-department" + blueprint_person = "fake-person" + integration_identifier = "smoke-test-integration" + smoke_test_suffix = environ.get("SMOKE_TEST_SUFFIX") + if smoke_test_suffix is not None: + integration_identifier = f"{integration_identifier}-{smoke_test_suffix}" + blueprint_person = f"{blueprint_person}-{smoke_test_suffix}" + blueprint_department = f"{blueprint_department}-{smoke_test_suffix}" + + return SmokeTestDetails( + integration_identifier=integration_identifier, + blueprint_person=blueprint_person, + blueprint_department=blueprint_department, + integration_version="0.1.4-dev", + integration_type="smoke-test", + ) + + +async def cleanup_smoke_test() -> None: + smoke_test_details = get_smoke_test_details() + client_id = environ.get("PORT_CLIENT_ID") + client_secret = environ.get("PORT_CLIENT_SECRET") + + if not client_secret or not client_id: + assert False, "Missing port credentials" + + base_url = environ.get("PORT_BASE_URL") + client = get_port_client_for_integration( + client_id, + client_secret, + smoke_test_details.integration_identifier, + smoke_test_details.integration_type, + smoke_test_details.integration_version, + base_url, + ) + + logger.info("Cleaning up fake integration") + await cleanup_integration( + client, + [smoke_test_details.blueprint_department, smoke_test_details.blueprint_person], + ) + logger.info("Cleaning up fake integration complete") + + +def get_port_client_for_fake_integration() -> PortClient: + smoke_test_details = get_smoke_test_details() + client_id = environ.get("PORT_CLIENT_ID") + client_secret = environ.get("PORT_CLIENT_SECRET") + + if not client_secret or not client_id: + assert False, "Missing port credentials" + + base_url = environ.get("PORT_BASE_URL") + client = get_port_client_for_integration( + client_id, + client_secret, + smoke_test_details.integration_identifier, + smoke_test_details.integration_type, + smoke_test_details.integration_version, + base_url, + ) + + return client diff --git a/port_ocean/tests/test_smoke.py b/port_ocean/tests/test_smoke.py index 4aac8f0076..a405705f2a 100644 --- a/port_ocean/tests/test_smoke.py +++ b/port_ocean/tests/test_smoke.py @@ -4,7 +4,7 @@ from port_ocean.clients.port.client import PortClient from port_ocean.clients.port.types import UserAgentType -from port_ocean.tests.helpers.fixtures import SmokeTestDetails +from port_ocean.tests.helpers.smoke_test import SmokeTestDetails pytestmark = pytest.mark.smoke diff --git a/pyproject.toml b/pyproject.toml index 1a837da4be..54749ff3d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.11.0" +version = "0.12.0" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io" @@ -176,3 +176,6 @@ exclude = ''' asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" addopts = "-vv -n auto --durations=10 --color=yes --ignore-glob='./integrations/*' ./port_ocean/tests" +markers = [ + "smoke: Smoke tests (deselect with '-m \"not smoke\"')" +] diff --git a/scripts/clean-smoke-test.py b/scripts/clean-smoke-test.py new file mode 100755 index 0000000000..e79b8f12e9 --- /dev/null +++ b/scripts/clean-smoke-test.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import asyncio +from port_ocean.tests.helpers.smoke_test import cleanup_smoke_test + + +async def main() -> None: + await cleanup_smoke_test() + + +asyncio.get_event_loop().run_until_complete(main()) diff --git a/scripts/run-smoke-test.sh b/scripts/run-smoke-test.sh index eccf618920..c24a3fa14d 100755 --- a/scripts/run-smoke-test.sh +++ b/scripts/run-smoke-test.sh @@ -42,6 +42,8 @@ if [[ $? != 0 ]]; then fi TAR_FILE=$(basename "${TAR_FULL_PATH}") +FAKE_INTEGRATION_VERSION=$(grep -E '^version = ".*"' "${ROOT_DIR}/integrations/fake-integration/pyproject.toml" | cut -d'"' -f2) + echo "Found release ${TAR_FILE}, triggering fake integration with ID: '${INTEGRATION_IDENTIFIER}'" # NOTE: Runs the fake integration with the modified blueprints and install the current core for a single sync @@ -56,5 +58,5 @@ docker run --rm -i \ -e OCEAN__INTEGRATION__TYPE="smoke-test" \ -e OCEAN__INTEGRATION__IDENTIFIER="${INTEGRATION_IDENTIFIER}" \ --name=ZOMG-TEST \ - ghcr.io/port-labs/port-ocean-fake-integration:0.1.1-dev \ + "ghcr.io/port-labs/port-ocean-fake-integration:${FAKE_INTEGRATION_VERSION}" \ -c "pip install --root-user-action=ignore /opt/dist/${TAR_FILE}[cli] && ocean sail -O"