From b3be4ced1dd0f9ad5b9a971c78782ab441f48073 Mon Sep 17 00:00:00 2001 From: Daniel Lopes Date: Wed, 2 Oct 2024 09:33:56 -0300 Subject: [PATCH 1/4] [docker build] update to add 2.x files and normalize lib versions --- Dockerfile | 73 +++++++++++++++++++++++++++++++++++--------- requirements-dev.txt | 30 +++++++++--------- requirements.txt | 22 ++++++------- 3 files changed, 85 insertions(+), 40 deletions(-) diff --git a/Dockerfile b/Dockerfile index a8c625f..208080e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,65 @@ +# Stage 1: Builder FROM python:3.11-bookworm AS builder -RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake && apt-get -y clean && mkdir -p /app/ && python3 -m venv /app/.venv -RUN rm /bin/sh && ln -s /bin/bash /bin/sh -COPY requirements.txt /app/ -RUN cd app && source .venv/bin/activate && pip3 install -r requirements.txt +# Set environment variables for paths to avoid repetition +ENV APP_DIR=/app \ + VENV_DIR=/app/.venv +# Update system and install required packages in a single RUN command to reduce layers +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ + cmake \ + build-essential \ + libatlas-base-dev \ + gfortran \ + libssl-dev \ + libffi-dev \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + libcurl4-openssl-dev \ + libboost-all-dev \ + libprotobuf-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p $APP_DIR \ + && python3 -m venv $VENV_DIR + +# Copy the requirements file first to leverage Docker caching in case dependencies don't change +COPY requirements.txt $APP_DIR/ + +# Activate virtual environment and install Python dependencies +# Use the full path for activation +RUN /bin/bash -c "source $VENV_DIR/bin/activate && pip3 install --no-cache-dir -r $APP_DIR/requirements.txt" + +# Stage 2: Runtime Image FROM python:3.11-bookworm AS runtime-image -RUN apt-get update && apt-get -y upgrade && apt-get -y clean -RUN useradd --create-home --shell /bin/sh --uid 8000 opencost -COPY --from=builder /app /app -COPY src/opencost_parquet_exporter.py /app/opencost_parquet_exporter.py -COPY src/data_types.json /app/data_types.json -COPY src/rename_cols.json /app/rename_cols.json -COPY src/ignore_alloc_keys.json /app/ignore_alloc_keys.json -RUN chmod 755 /app/opencost_parquet_exporter.py && chown -R opencost /app/ + +# Set environment variables for paths +ENV APP_DIR=/app \ + VENV_DIR=/app/.venv \ + PATH="$VENV_DIR/bin:$PATH" + +# Update and clean up in a single step to minimize image size +RUN apt-get update && apt-get -y upgrade \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user for security +RUN useradd --create-home --shell /bin/sh --uid 8000 opencost + +# Copy application files from the builder stage +COPY --from=builder $APP_DIR $APP_DIR + +# Copy all files from /src to /app +COPY src/ $APP_DIR/ + +# Set correct permissions for the application files +RUN chmod 755 $APP_DIR/opencost_parquet_exporter.py \ + && chown -R opencost $APP_DIR + +# Switch to the non-root user USER opencost -ENV PATH="/app/.venv/bin:$PATH" -CMD ["/app/opencost_parquet_exporter.py"] + +# Default entrypoint and command ENTRYPOINT ["/app/.venv/bin/python3"] +CMD ["/app/opencost_parquet_exporter.py"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 878b07d..c784c73 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,15 +1,15 @@ -numpy==1.26.3 -pandas==2.1.4 -boto3==1.34.21 -requests==2.32.0 -python-dateutil==2.8.2 -pytz==2023.3.post1 -six==1.16.0 -tzdata==2023.4 -pyarrow==14.0.1 -azure-storage-blob==12.19.1 -azure-identity==1.16.1 -# The dependencies bellow are only used for development. -freezegun==1.4.0 -pylint==3.0.3 -pytest==7.4.4 +numpy>=1.26.3 +pandas>=2.1.4 +boto3>=1.34.21 +requests>=2.32.0 +python-dateutil>=2.8.2 +pytz>=2023.3.post1 +six>=1.16.0 +tzdata>=2023.4 +pyarrow>=14.0.1 +azure-storage-blob>=12.19.1 +azure-identity>=1.16.1 +# The dependencies below are only used for development. +freezegun>=1.4.0 +pylint>=3.0.3 +pytest>=7.4.4 diff --git a/requirements.txt b/requirements.txt index bbead29..7a83a26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -numpy==1.26.3 -pandas==2.1.4 -boto3==1.34.21 -requests==2.32.0 -python-dateutil==2.8.2 -pytz==2023.3.post1 -six==1.16.0 -tzdata==2023.4 -pyarrow==14.0.1 -azure-storage-blob==12.19.1 -azure-identity==1.16.1 \ No newline at end of file +numpy>=1.26.3 +pandas>=2.1.4 +boto3>=1.34.21 +requests>=2.32.0 +python-dateutil>=2.8.2 +pytz>=2023.3.post1 +six>=1.16.0 +tzdata>=2023.4 +pyarrow>=14.0.1 +azure-storage-blob>=12.19.1 +azure-identity>=1.16.1 \ No newline at end of file From f3d05136a5af90f8ecd07ed422a293074a92e9c6 Mon Sep 17 00:00:00 2001 From: Daniel Lopes Date: Wed, 2 Oct 2024 09:34:29 -0300 Subject: [PATCH 2/4] [kubernetes] fix cronjob CR --- examples/k8s_cron_job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/k8s_cron_job.yaml b/examples/k8s_cron_job.yaml index 8930c9d..ef86a02 100644 --- a/examples/k8s_cron_job.yaml +++ b/examples/k8s_cron_job.yaml @@ -55,6 +55,7 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File command: ["/app/.venv/bin/python3"] # Update this is if the ENTRYPOINT changes + args: ["/app/opencost_parquet_exporter.py"] # Update this is if the ENTRYPOINT changes dnsConfig: options: - name: single-request-reopen From b772e4da4b1109bcfeff5c21f7b22eea1228da8e Mon Sep 17 00:00:00 2001 From: Daniel Lopes Date: Wed, 2 Oct 2024 09:41:54 -0300 Subject: [PATCH 3/4] [requirements.txt] lock lib versions --- requirements-dev.txt | 18 +++++++++--------- requirements.txt | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c784c73..c5f270c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,12 @@ -numpy>=1.26.3 -pandas>=2.1.4 -boto3>=1.34.21 -requests>=2.32.0 -python-dateutil>=2.8.2 -pytz>=2023.3.post1 -six>=1.16.0 -tzdata>=2023.4 -pyarrow>=14.0.1 +numpy==2.1.1 +pandas==2.2.3 +boto3==1.35.31 +requests==2.32.3 +python-dateutil==2.9.0.post0 +pytz==2024.2 +six==1.16.0 +tzdata==2024.2 +pyarrow==17.0.0 azure-storage-blob>=12.19.1 azure-identity>=1.16.1 # The dependencies below are only used for development. diff --git a/requirements.txt b/requirements.txt index 7a83a26..01a4acc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -numpy>=1.26.3 -pandas>=2.1.4 -boto3>=1.34.21 -requests>=2.32.0 -python-dateutil>=2.8.2 -pytz>=2023.3.post1 -six>=1.16.0 -tzdata>=2023.4 -pyarrow>=14.0.1 +numpy==2.1.1 +pandas==2.2.3 +boto3==1.35.31 +requests==2.32.3 +python-dateutil==2.9.0.post0 +pytz==2024.2 +six==1.16.0 +tzdata==2024.2 +pyarrow==17.0.0 azure-storage-blob>=12.19.1 azure-identity>=1.16.1 \ No newline at end of file From 4e8e4e3ddc11835f06d1c354b80537f19b19a85b Mon Sep 17 00:00:00 2001 From: Daniel Lopes Date: Wed, 2 Oct 2024 14:38:16 -0300 Subject: [PATCH 4/4] [kubernetes] migrate image base from python:3.11-bookworm to python:3.12-bookworm to support multi-arch --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 208080e..2a3c4b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Builder -FROM python:3.11-bookworm AS builder +FROM python:3.12-bookworm AS builder # Set environment variables for paths to avoid repetition ENV APP_DIR=/app \ @@ -32,7 +32,7 @@ COPY requirements.txt $APP_DIR/ RUN /bin/bash -c "source $VENV_DIR/bin/activate && pip3 install --no-cache-dir -r $APP_DIR/requirements.txt" # Stage 2: Runtime Image -FROM python:3.11-bookworm AS runtime-image +FROM python:3.12-bookworm AS runtime-image # Set environment variables for paths ENV APP_DIR=/app \