Move to alpine and improve security & smoke test

.github/workflows/core-test.yml

@@ -29,16 +29,16 @@ jobs:
         run: |
           make install
 
-      - name: Build core for smoke test
-        run: |
-          make build
-
       - name: Unit Test Core
         env:
           PYTEST_ADDOPTS: --junitxml=junit/unit-test-results-ocean/core.xml
         run: |
           make test
 
+      - name: Build core for smoke test
+        run: |
+          make build
+
       - name: Run fake integration for core test
         env:
           PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
@@ -58,6 +58,16 @@ jobs:
         run: |
           make test/smoke
 
+      - name: Cleanup Smoke Test
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          make test/smoke
+
       - name: Install current core for all integrations
         run: |
           echo "Installing local core for all integrations"

.github/workflows/detect-changes-matrix.yml

@@ -41,6 +41,7 @@ jobs:
             integrations:
               - 'integrations/**'
               - '!integrations/**/*.md'
+              - '!integrations/_infra/*'
 
       - name: Set integrations and all matrix
         id: set-all-matrix

Makefile

@@ -43,7 +43,7 @@ define deactivate_virtualenv
     fi
 endef
 
-.SILENT: install install/all test/all lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all
+.SILENT: install install/all test/all test/smoke clean/smoke lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all
 
 
 # Install dependencies
@@ -115,11 +115,14 @@ new:
 	$(ACTIVATE) && poetry run ocean new ./integrations --public
 
 test:
-	$(ACTIVATE) && pytest
+	$(ACTIVATE) && pytest -m 'not smoke'
 
 test/smoke:
 	$(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} pytest -m smoke
 
+clean/smoke:
+	$(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./scripts/clean-smoke-test.py
+
 test/watch:
 	$(ACTIVATE) && \
 		pytest \

integrations/_infra/Dockerfile

@@ -1,28 +1,72 @@
-FROM python:3.11-slim-bookworm
+FROM python:3.11-alpine AS base
 
 ARG BUILD_CONTEXT
+
+ENV LIBRDKAFKA_VERSION=1.9.2
+
+# Install system dependencies and libraries
+RUN apk add --no-cache \
+    gcc \
+    musl-dev \
+    build-base \
+    bash \
+    oniguruma-dev \
+    make \
+    autoconf \
+    automake \
+    libtool \
+    curl \
+    # librdkafka-dev \
+    libffi-dev \
+        # Install community librdkafka-dev since the default in alpine is older
+        && echo "@edge http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \
+        && echo "@edgecommunity http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+        && apk add --no-cache alpine-sdk "librdkafka@edgecommunity>=${LIBRDKAFKA_VERSION}" "librdkafka-dev@edgecommunity>=${LIBRDKAFKA_VERSION}" \
+        && curl -sSL https://install.python-poetry.org | python3 - \
+        && /root/.local/bin/poetry config virtualenvs.in-project true
+
+
+WORKDIR /app
+
+COPY ./${BUILD_CONTEXT}/pyproject.toml ./${BUILD_CONTEXT}/poetry.lock /app/
+
+RUN /root/.local/bin/poetry install --without dev --no-root --no-interaction --no-ansi --no-cache && pip cache purge
+
+FROM python:3.11-alpine AS prod
+
 ARG INTEGRATION_VERSION
+ARG BUILD_CONTEXT
 
 LABEL INTEGRATION_VERSION=${INTEGRATION_VERSION}
 # Used to ensure that new integrations will be public, see https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility
-LABEL org.opencontainers.image.source https://github.com/port-labs/ocean
+LABEL org.opencontainers.image.source=https://github.com/port-labs/ocean
 
-ENV LIBRDKAFKA_VERSION 1.9.2
+# Install only runtime dependencies
+RUN apk add --no-cache \
+        librdkafka-dev \
+        bash \
+        oniguruma-dev \
+    # Install community librdkafka-dev since the default in alpine is older
+    && echo "@edge http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories \
+    && echo "@edgecommunity http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+    && apk add --no-cache alpine-sdk "librdkafka@edgecommunity>=${LIBRDKAFKA_VERSION}" "librdkafka-dev@edgecommunity>=${LIBRDKAFKA_VERSION}" \
+    && test -e /usr/local/share/ca-certificates/cert.crt && update-ca-certificates || true
 
 WORKDIR /app
 
-RUN apt update && \
-    apt install -y wget make g++ libssl-dev autoconf automake libtool curl librdkafka-dev && \
-    apt-get clean
-
-COPY ./integrations/_infra/init.sh /app/init.sh
-
-RUN chmod +x /app/init.sh
+# Copy dependencies from the build stage
+COPY --from=base /app /app
 
+# Copy the application code
 COPY ./${BUILD_CONTEXT} /app
 
-COPY ./integrations/_infra/Makefile /app/Makefile
-
-RUN export POETRY_VIRTUALENVS_CREATE=false && make install/prod && pip cache purge
+# Ensure that ocean is available for all in path
+RUN chmod a+x /app/.venv/bin/ocean \
+    && ln -s /app/.venv/bin/ocean /usr/bin/ocean \
+    #    # Fix security issues
+    && apk upgrade busybox --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main \
+    # Clean up old setuptools
+    && pip uninstall -y setuptools py3-setuptools
 
-ENTRYPOINT ./init.sh
+# Run the application
+CMD ["ocean", "sail"]

port_ocean/log/sensetive.py

@@ -21,7 +21,7 @@
     "GitHub": r"[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]",
     "Google Cloud Platform API Key": r"AIza[0-9A-Za-z\\-_]{35}",
     "Google Cloud Platform OAuth": r"[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com",
-    "Google (GCP) Service-account": r'"type": "service_account"',
+    "Google (GCP) Service-account": f'"type":{" "}"service_account"',
     "Google OAuth Access Token": r"ya29\\.[0-9A-Za-z\\-_]+",
     "Connection String": r"[a-zA-Z]+:\/\/[^/\s]+:[^/\s]+@[^/\s]+\/[^/\s]+",
 }

port_ocean/tests/helpers/fixtures.py

@@ -1,9 +1,8 @@
-from os import environ, path
-from typing import Any, AsyncGenerator, Callable, List, Tuple, Union
+from os import path
+from typing import Any, Callable, List, Tuple
 
+import pytest
 import pytest_asyncio
-from loguru import logger
-from pydantic import BaseModel
 
 from port_ocean.clients.port.client import PortClient
 from port_ocean.core.handlers.port_app_config.models import ResourceConfig
@@ -12,96 +11,19 @@
     get_integation_resource_configs,
     get_integration_ocean_app,
 )
+from port_ocean.tests.helpers.smoke_test import (
+    SmokeTestDetails,
+    get_port_client_for_fake_integration,
+    get_smoke_test_details,
+)
 
 
-def get_port_client_for_integration(
-    client_id: str,
-    client_secret: str,
-    integration_identifier: str,
-    integration_type: str,
-    integration_version: str,
-    base_url: Union[str, None],
-) -> PortClient:
-    return PortClient(
-        base_url=base_url or "https://api.getport/io",
-        client_id=client_id,
-        client_secret=client_secret,
-        integration_identifier=integration_identifier,
-        integration_type=integration_type,
-        integration_version=integration_version,
-    )
-
-
-async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None:
-    for blueprint in blueprints:
-        try:
-            bp = await client.get_blueprint(blueprint)
-            if bp is not None:
-                migration_id = await client.delete_blueprint(
-                    identifier=blueprint, delete_entities=True
-                )
-                if migration_id:
-                    await client.wait_for_migration_to_complete(
-                        migration_id=migration_id
-                    )
-        except Exception as bp_e:
-            logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}")
-    headers = await client.auth.headers()
-    try:
-        await client.client.delete(
-            f"{client.auth.api_url}/integrations/{client.integration_identifier}",
-            headers=headers,
-        )
-    except Exception as int_e:
-        logger.info(
-            f"Failed to delete integration ({client.integration_identifier}): {int_e}"
-        )
-
-
-class SmokeTestDetails(BaseModel):
-    integration_identifier: str
-    blueprint_department: str
-    blueprint_person: str
-
-
-@pytest_asyncio.fixture()
-async def port_client_for_fake_integration() -> (
-    AsyncGenerator[Tuple[SmokeTestDetails, PortClient], None]
-):
-    blueprint_department = "fake-department"
-    blueprint_person = "fake-person"
-    integration_identifier = "smoke-test-integration"
-    smoke_test_suffix = environ.get("SMOKE_TEST_SUFFIX")
-    client_id = environ.get("PORT_CLIENT_ID")
-    client_secret = environ.get("PORT_CLIENT_SECRET")
-
-    if not client_secret or not client_id:
-        assert False, "Missing port credentials"
-
-    base_url = environ.get("PORT_BASE_URL")
-    integration_version = "0.1.1-dev"
-    integration_type = "smoke-test"
-    if smoke_test_suffix is not None:
-        integration_identifier = f"{integration_identifier}-{smoke_test_suffix}"
-        blueprint_person = f"{blueprint_person}-{smoke_test_suffix}"
-        blueprint_department = f"{blueprint_department}-{smoke_test_suffix}"
-
-    client = get_port_client_for_integration(
-        client_id,
-        client_secret,
-        integration_identifier,
-        integration_type,
-        integration_version,
-        base_url,
-    )
+@pytest.fixture
+def port_client_for_fake_integration() -> Tuple[SmokeTestDetails, PortClient]:
+    smoke_test_details = get_smoke_test_details()
+    port_client = get_port_client_for_fake_integration()
 
-    smoke_test_details = SmokeTestDetails(
-        integration_identifier=integration_identifier,
-        blueprint_person=blueprint_person,
-        blueprint_department=blueprint_department,
-    )
-    yield smoke_test_details, client
-    await cleanup_integration(client, [blueprint_department, blueprint_person])
+    return smoke_test_details, port_client
 
 
 @pytest_asyncio.fixture

port_ocean/tests/helpers/integration.py

@@ -0,0 +1,31 @@
+from typing import List
+
+from loguru import logger
+
+from port_ocean.clients.port.client import PortClient
+
+
+async def cleanup_integration(client: PortClient, blueprints: List[str]) -> None:
+    for blueprint in blueprints:
+        try:
+            bp = await client.get_blueprint(blueprint)
+            if bp is not None:
+                migration_id = await client.delete_blueprint(
+                    identifier=blueprint, delete_entities=True
+                )
+                if migration_id:
+                    await client.wait_for_migration_to_complete(
+                        migration_id=migration_id
+                    )
+        except Exception as bp_e:
+            logger.info(f"Skipping missing blueprint ({blueprint}): {bp_e}")
+    headers = await client.auth.headers()
+    try:
+        await client.client.delete(
+            f"{client.auth.api_url}/integrations/{client.integration_identifier}",
+            headers=headers,
+        )
+    except Exception as int_e:
+        logger.info(
+            f"Failed to delete integration ({client.integration_identifier}): {int_e}"
+        )

port_ocean/tests/helpers/port_client.py

@@ -0,0 +1,21 @@
+from typing import Union
+
+from port_ocean.clients.port.client import PortClient
+
+
+def get_port_client_for_integration(
+    client_id: str,
+    client_secret: str,
+    integration_identifier: str,
+    integration_type: str,
+    integration_version: str,
+    base_url: Union[str, None],
+) -> PortClient:
+    return PortClient(
+        base_url=base_url or "https://api.getport/io",
+        client_id=client_id,
+        client_secret=client_secret,
+        integration_identifier=integration_identifier,
+        integration_type=integration_type,
+        integration_version=integration_version,
+    )