Simplify docker image building process (#954)

* Simplify docker image building process

* Format & cleanup

.github/workflows/cd.yml

@@ -146,14 +146,6 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: ${{ env.POETRY_VERSION }}
-
-      - name: Export requirements.txt
-        run: poetry export -E all -f requirements.txt --output requirements.txt
-
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:

Dockerfile

@@ -1,20 +1,32 @@
+FROM python:3.8-slim AS base
 
-FROM python:3.8-slim
+# Builder - Dependencies
+FROM base AS builder
 
-RUN apt-get clean
-RUN apt-get update
-RUN apt-get install -y git build-essential libsasl2-dev
+RUN pip install poetry
 
-COPY ./requirements.txt /src/requirements.txt
+COPY pyproject.toml poetry.lock src/
+WORKDIR /src
 
-RUN pip install -r /src/requirements.txt --no-deps
+RUN poetry export \
+    --without dev \
+    --all-extras \
+    --format=requirements.txt \
+    --output=requirements.txt
 
-RUN rm -rf /src
+# Runtime
+FROM base AS runtime
 
-COPY . /src
+RUN apt-get clean
+RUN apt-get update
+RUN apt-get install -y git build-essential libsasl2-dev
 
-RUN pip install '/src[all]'
+COPY --from=builder /src/requirements.txt /dep/requirements.txt
+RUN pip install -r /dep/requirements.txt --no-deps
+RUN rm -rf /dep
 
+COPY . src/
+RUN pip install '/src[all]'
 RUN rm -rf /src
 
 CMD ["sh", "-c", "metaphor ${CONNECTOR} ${CONFIG_FILE}"]

docs/docker.md

@@ -8,7 +8,7 @@ We follow [Semantic Versioning](https://semver.org/) and [docker tagging best pr
 
 ## Running
 
-The Docker image contains all the required dependencies so running it should be very straightforward. Note that you'll need to provide your local config file to the Docker container via [bind mounts](https://docs.docker.com/storage/bind-mounts/). For example,
+The Docker image contains all the required dependencies so running it should be very straightforward. Note that you'll need to provide your local config file to the Docker container via [bind mounts](https://docs.docker.com/storage/bind-mounts/). For example, this command passes your local config file (`~/config.yml`) to the docker image,
 
 ```sh
 docker run -it --rm \
@@ -17,7 +17,7 @@ docker run -it --rm \
   metaphor bigquery /config.yml
 ```
 
-If you configure to write the [output](../metaphor/common/docs/output.md) to a local directory instead of an S3 path, you'll also need to bind mount the output directory to access the files, e.g.
+If you configure to write the [output](../metaphor/common/docs/output.md) to a local directory instead of an S3 path, you'll also need to bind mount the output directory (`~/output` in this case) to access the files, e.g.
 
 ```yml
 # ~/config.yml
@@ -45,10 +45,6 @@ To build and publish an image with the `test` tag for testing, run the following
 # Assuming not yet logged in
 docker login
 
-# Our Dockerfile refereneces requirements.txt to decrease build time. Make sure
-# to run this before running `docker build`.
-poetry export --all-extras --format=requirements.txt --output=requirements.txt
-
 docker build --platform linux/amd64 -t metaphordata/connectors:test .
 docker push metaphordata/connectors:test
 ```

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.78"
+version = "0.14.79"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]