From 7a45940792a2567b0e176e0ed747474fa1d7dc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 12 Jul 2024 15:39:35 +0200 Subject: [PATCH 1/4] Attempt to fix the problem with e2e tests --- .../hosted_inference_e2e_test_production.yml | 2 +- .../hosted_inference_e2e_test_staging.yml | 2 +- .../hosted_platform_tests/conftest.py | 23 +++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hosted_inference_e2e_test_production.yml b/.github/workflows/hosted_inference_e2e_test_production.yml index 5fcbdff05..4e1f71912 100644 --- a/.github/workflows/hosted_inference_e2e_test_production.yml +++ b/.github/workflows/hosted_inference_e2e_test_production.yml @@ -24,7 +24,7 @@ jobs: - name: 🦾 Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements/requirements.test.unit.txt -r requirements/requirements.test.integration.txt + python -m pip install -r requirements/requirements.test.unit.txt -r requirements/requirements.test.integration.txt -r requirements/requirements.sdk.http.txt - name: 📝 E2E test of HOSTED INFERENCE at 🚨 PRODUCTION 🚨 🔥🔥🔥🔥 run: SKIP_WARMUP=${{ github.event.inputs.skip_warmup }} HOSTED_PLATFORM_TESTS_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} HOSTED_PLATFORM_TESTS_PROJECT=roboflow-platform pytest tests/inference/hosted_platform_tests/ diff --git a/.github/workflows/hosted_inference_e2e_test_staging.yml b/.github/workflows/hosted_inference_e2e_test_staging.yml index 73b0c44ac..cb904160a 100644 --- a/.github/workflows/hosted_inference_e2e_test_staging.yml +++ b/.github/workflows/hosted_inference_e2e_test_staging.yml @@ -24,7 +24,7 @@ jobs: - name: 🦾 Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements/requirements.test.unit.txt -r requirements/requirements.test.integration.txt + python -m pip install -r requirements/requirements.test.unit.txt -r requirements/requirements.test.integration.txt -r requirements/requirements.sdk.http.txt - name: 📝 E2E test of HOSTED INFERENCE at 😎 STAGING 😎 🔥🔥🔥🔥 run: SKIP_WARMUP=${{ github.event.inputs.skip_warmup }} HOSTED_PLATFORM_TESTS_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} HOSTED_PLATFORM_TESTS_PROJECT=roboflow-staging pytest tests/inference/hosted_platform_tests/ diff --git a/tests/inference/hosted_platform_tests/conftest.py b/tests/inference/hosted_platform_tests/conftest.py index 1712de114..0d9ef9af0 100644 --- a/tests/inference/hosted_platform_tests/conftest.py +++ b/tests/inference/hosted_platform_tests/conftest.py @@ -2,13 +2,31 @@ import os from enum import Enum from functools import partial +from typing import Any import numpy as np import pytest -from inference.core.utils.environment import str2bool from inference_sdk import InferenceHTTPClient + +def str2bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if not issubclass(type(value), str): + raise ValueError( + f"Expected a boolean environment variable (true or false) but got '{value}'" + ) + if value.lower() == "true": + return True + elif value.lower() == "false": + return False + else: + raise ValueError( + f"Expected a boolean environment variable (true or false) but got '{value}'" + ) + + MAXIMUM_CONSECUTIVE_INVOCATION_ERRORS = int( os.getenv("HOSTED_PLATFORM_TESTS_MAX_WARMUP_CONSECUTIVE_ERRORS", 5) ) @@ -16,6 +34,7 @@ os.getenv("HOSTED_PLATFORM_TESTS_MIN_WARMUP_SUCCESS_RESPONSES", 5) ) SKIP_WARMUP = str2bool(os.getenv("SKIP_WARMUP", False)) +IMAGE_URL = "https://media.roboflow.com/inference/dog.jpeg" class PlatformEnvironment(Enum): @@ -232,4 +251,4 @@ def retry_at_max_n_times(function: callable, n: int, function_description: str) raise Exception(f"Could not achieve success of {function_description}") -IMAGE_URL = "https://media.roboflow.com/inference/dog.jpeg" + From 5ee1a6ce749b539da37df4c0d25b0e5cb72d2a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 12 Jul 2024 15:58:30 +0200 Subject: [PATCH 2/4] Fix build & push for hosted API --- .../workflows/prod_build_and_push_hosted_inference_image.yml | 4 ++-- .../staging_build_and_push_hosted_inference_image.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prod_build_and_push_hosted_inference_image.yml b/.github/workflows/prod_build_and_push_hosted_inference_image.yml index c0b8d7f40..5245e44b5 100644 --- a/.github/workflows/prod_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/prod_build_and_push_hosted_inference_image.yml @@ -32,10 +32,10 @@ jobs: registries: ${{ secrets.PRODUCTION_AWS_ACCOUNT_ID }} mask-password: "true" # see: https://github.com/aws-actions/amazon-ecr-login#docker-credentials - name: 🛠️ Build and push docker image [SLIM] to PRODUCTION 🚨 - if: ${{ !github.event.inputs.exclude_slim_lambda }} + if: ${{ github.event.inputs.exclude_slim_lambda == 'true' }} run: | ./docker/publish/onnx_lambda.sh roboflow-platform slim - name: 🛠️ Build and push docker image [CORE MODELS] to PRODUCTION 🚨 - if: ${{ !github.event.inputs.exclude_core_models_lambda }} + if: ${{ !github.event.inputs.exclude_core_models_lambda == 'true' }} run: | ./docker/publish/onnx_lambda.sh roboflow-platform diff --git a/.github/workflows/staging_build_and_push_hosted_inference_image.yml b/.github/workflows/staging_build_and_push_hosted_inference_image.yml index a88e54302..0254e8fb4 100644 --- a/.github/workflows/staging_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/staging_build_and_push_hosted_inference_image.yml @@ -33,10 +33,10 @@ jobs: registries: ${{ secrets.STAGING_AWS_ACCOUNT_ID }} mask-password: "true" # see: https://github.com/aws-actions/amazon-ecr-login#docker-credentials - name: 🛠️ Build and push docker image [SLIM] to STAGING 😎 - if: ${{ !github.event.inputs.exclude_slim_lambda }} + if: ${{ github.event.inputs.exclude_slim_lambda == 'true' }} run: | ./docker/publish/onnx_lambda.sh roboflow-staging slim - name: 🛠️ Build and push docker image [CORE MODELS] to STAGING 😎 - if: ${{ !github.event.inputs.exclude_core_models_lambda }} + if: ${{ !github.event.inputs.exclude_core_models_lambda == 'true' }} run: | ./docker/publish/onnx_lambda.sh roboflow-staging From ad9373cb2ab11cc7071b0c54bb67629f7fef9532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 12 Jul 2024 16:00:16 +0200 Subject: [PATCH 3/4] Fix build & push for hosted API --- .../workflows/prod_build_and_push_hosted_inference_image.yml | 4 ++-- .../staging_build_and_push_hosted_inference_image.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prod_build_and_push_hosted_inference_image.yml b/.github/workflows/prod_build_and_push_hosted_inference_image.yml index 5245e44b5..ea923911f 100644 --- a/.github/workflows/prod_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/prod_build_and_push_hosted_inference_image.yml @@ -32,10 +32,10 @@ jobs: registries: ${{ secrets.PRODUCTION_AWS_ACCOUNT_ID }} mask-password: "true" # see: https://github.com/aws-actions/amazon-ecr-login#docker-credentials - name: 🛠️ Build and push docker image [SLIM] to PRODUCTION 🚨 - if: ${{ github.event.inputs.exclude_slim_lambda == 'true' }} + if: ${{ github.event.inputs.exclude_slim_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-platform slim - name: 🛠️ Build and push docker image [CORE MODELS] to PRODUCTION 🚨 - if: ${{ !github.event.inputs.exclude_core_models_lambda == 'true' }} + if: ${{ !github.event.inputs.exclude_core_models_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-platform diff --git a/.github/workflows/staging_build_and_push_hosted_inference_image.yml b/.github/workflows/staging_build_and_push_hosted_inference_image.yml index 0254e8fb4..5b6a81562 100644 --- a/.github/workflows/staging_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/staging_build_and_push_hosted_inference_image.yml @@ -33,10 +33,10 @@ jobs: registries: ${{ secrets.STAGING_AWS_ACCOUNT_ID }} mask-password: "true" # see: https://github.com/aws-actions/amazon-ecr-login#docker-credentials - name: 🛠️ Build and push docker image [SLIM] to STAGING 😎 - if: ${{ github.event.inputs.exclude_slim_lambda == 'true' }} + if: ${{ github.event.inputs.exclude_slim_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-staging slim - name: 🛠️ Build and push docker image [CORE MODELS] to STAGING 😎 - if: ${{ !github.event.inputs.exclude_core_models_lambda == 'true' }} + if: ${{ !github.event.inputs.exclude_core_models_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-staging From 2b5ddacf48ee8b8e25820b29877f8b7d5c8c1879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Fri, 12 Jul 2024 16:04:11 +0200 Subject: [PATCH 4/4] Fix build & push for hosted API --- .../workflows/prod_build_and_push_hosted_inference_image.yml | 2 +- .../workflows/staging_build_and_push_hosted_inference_image.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prod_build_and_push_hosted_inference_image.yml b/.github/workflows/prod_build_and_push_hosted_inference_image.yml index ea923911f..36313b975 100644 --- a/.github/workflows/prod_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/prod_build_and_push_hosted_inference_image.yml @@ -36,6 +36,6 @@ jobs: run: | ./docker/publish/onnx_lambda.sh roboflow-platform slim - name: 🛠️ Build and push docker image [CORE MODELS] to PRODUCTION 🚨 - if: ${{ !github.event.inputs.exclude_core_models_lambda == 'false' }} + if: ${{ github.event.inputs.exclude_core_models_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-platform diff --git a/.github/workflows/staging_build_and_push_hosted_inference_image.yml b/.github/workflows/staging_build_and_push_hosted_inference_image.yml index 5b6a81562..8e964a727 100644 --- a/.github/workflows/staging_build_and_push_hosted_inference_image.yml +++ b/.github/workflows/staging_build_and_push_hosted_inference_image.yml @@ -37,6 +37,6 @@ jobs: run: | ./docker/publish/onnx_lambda.sh roboflow-staging slim - name: 🛠️ Build and push docker image [CORE MODELS] to STAGING 😎 - if: ${{ !github.event.inputs.exclude_core_models_lambda == 'false' }} + if: ${{ github.event.inputs.exclude_core_models_lambda == 'false' }} run: | ./docker/publish/onnx_lambda.sh roboflow-staging