diff --git a/Dockerfile b/Dockerfile
index 4701510..2fced88 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,17 +6,22 @@ FROM --platform=linux/amd64 ghcr.io/owl-corp/python-poetry-base:$python_version
 RUN apt-get update && apt-get install --no-install-recommends -y libldap2-dev libsasl2-dev gcc && apt autoclean && rm -rf /var/lib/apt/lists/*
 
 COPY pyproject.toml poetry.lock ./
-RUN poetry install --without dev --no-root
+
+# Only the LDAP group has deps that requires gcc, so only build that for now
+RUN poetry install --only ldap --no-root
 
 FROM --platform=linux/amd64 ghcr.io/owl-corp/python-poetry-base:$python_version
 
 RUN apt-get update && apt-get install --no-install-recommends -y libmagickwand-dev && rm -rf /var/lib/apt/lists/*
 
-# Install project dependencies
 WORKDIR /app
+
+# Copy the pre-built LDAP group from the build stage that required gcc
 COPY --from=wheel-builder /opt/poetry/cache /opt/poetry/cache
+
+# Install the rest of the dependencies
 COPY pyproject.toml poetry.lock ./
-RUN poetry install --without dev --no-root
+RUN poetry install --without dev --with ldap --no-root
 
 # Set Git SHA environment variable for Sentry
 ARG git_sha="development"
diff --git a/poetry.lock b/poetry.lock
index 0f05575..ed1a4af 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1821,4 +1821,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "3.12.*"
-content-hash = "c7bde0effe7684b430552a574540255d6a54331c91d22957e54d6985f49c5161"
+content-hash = "fdbe86971f80596ba6eee9e3b98a51ee6cba2f7d288ea37d7fc9b13e49ab7f99"
diff --git a/pyproject.toml b/pyproject.toml
index 465683a..8968ead 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,8 @@ pre-commit = "3.7.1"
 ruff = "0.5.4"
 taskipy = "1.13.0"
 
+[tool.poetry.group.ldap]
+optional = true
 [tool.poetry.group.ldap.dependencies]
 bonsai = "^1.5.3"