From e4d25b49b4e68e921a4f838d01eaadd7979ab32a Mon Sep 17 00:00:00 2001 From: Mars Lan Date: Sun, 18 Aug 2024 15:08:05 -0700 Subject: [PATCH] Simplify docker image building process (#954) * Simplify docker image building process * Format & cleanup --- .github/workflows/cd.yml | 8 -------- Dockerfile | 30 +++++++++++++++++++++--------- docs/docker.md | 8 ++------ pyproject.toml | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dfc528f5..b15feb2c 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -146,14 +146,6 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - name: Setup Poetry - uses: abatilo/actions-poetry@v2.1.4 - with: - poetry-version: ${{ env.POETRY_VERSION }} - - - name: Export requirements.txt - run: poetry export -E all -f requirements.txt --output requirements.txt - - name: Login to Docker Hub uses: docker/login-action@v1 with: diff --git a/Dockerfile b/Dockerfile index 4225b488..4e2ba73c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,32 @@ +FROM python:3.8-slim AS base -FROM python:3.8-slim +# Builder - Dependencies +FROM base AS builder -RUN apt-get clean -RUN apt-get update -RUN apt-get install -y git build-essential libsasl2-dev +RUN pip install poetry -COPY ./requirements.txt /src/requirements.txt +COPY pyproject.toml poetry.lock src/ +WORKDIR /src -RUN pip install -r /src/requirements.txt --no-deps +RUN poetry export \ + --without dev \ + --all-extras \ + --format=requirements.txt \ + --output=requirements.txt -RUN rm -rf /src +# Runtime +FROM base AS runtime -COPY . /src +RUN apt-get clean +RUN apt-get update +RUN apt-get install -y git build-essential libsasl2-dev -RUN pip install '/src[all]' +COPY --from=builder /src/requirements.txt /dep/requirements.txt +RUN pip install -r /dep/requirements.txt --no-deps +RUN rm -rf /dep +COPY . src/ +RUN pip install '/src[all]' RUN rm -rf /src CMD ["sh", "-c", "metaphor ${CONNECTOR} ${CONFIG_FILE}"] diff --git a/docs/docker.md b/docs/docker.md index ecf53ae9..f07cf288 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -8,7 +8,7 @@ We follow [Semantic Versioning](https://semver.org/) and [docker tagging best pr ## Running -The Docker image contains all the required dependencies so running it should be very straightforward. Note that you'll need to provide your local config file to the Docker container via [bind mounts](https://docs.docker.com/storage/bind-mounts/). For example, +The Docker image contains all the required dependencies so running it should be very straightforward. Note that you'll need to provide your local config file to the Docker container via [bind mounts](https://docs.docker.com/storage/bind-mounts/). For example, this command passes your local config file (`~/config.yml`) to the docker image, ```sh docker run -it --rm \ @@ -17,7 +17,7 @@ docker run -it --rm \ metaphor bigquery /config.yml ``` -If you configure to write the [output](../metaphor/common/docs/output.md) to a local directory instead of an S3 path, you'll also need to bind mount the output directory to access the files, e.g. +If you configure to write the [output](../metaphor/common/docs/output.md) to a local directory instead of an S3 path, you'll also need to bind mount the output directory (`~/output` in this case) to access the files, e.g. ```yml # ~/config.yml @@ -45,10 +45,6 @@ To build and publish an image with the `test` tag for testing, run the following # Assuming not yet logged in docker login -# Our Dockerfile refereneces requirements.txt to decrease build time. Make sure -# to run this before running `docker build`. -poetry export --all-extras --format=requirements.txt --output=requirements.txt - docker build --platform linux/amd64 -t metaphordata/connectors:test . docker push metaphordata/connectors:test ``` diff --git a/pyproject.toml b/pyproject.toml index 2fa37e28..8205bbdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metaphor-connectors" -version = "0.14.78" +version = "0.14.79" license = "Apache-2.0" description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app." authors = ["Metaphor "]