Optimize Docker builds (#5918)

We use Docker cache mounts to accelerate Docker builds with Go. With cache mounts, we can cache package dependencies and the Go build cache in a persistent fashion (i.e., persists across builds). As a minor optimization, we also use a bind mount for go.mod & go.sum, to avoid a COPY statement and an extra layer. We could do the same thing for the antrea source directory, but it would add extra complexity to the Dockerfiles, with no clear benefit (~2s per build). With these changes, an incremental builds using ./hack/build-antrea-linux-all.sh takes less than a minute (instead of more than 5 minutes). Signed-off-by: Antonin Bas <[email protected]>
antrea-io · Jan 25, 2024 · 538e5f6 · 538e5f6
1 parent 3f267bb
commit 538e5f6
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 36 deletions.
diff --git a/build/images/Dockerfile.build.agent.ubi b/build/images/Dockerfile.build.agent.ubi
@@ -39,18 +39,22 @@ ENV PATH /usr/local/go/bin:$PATH
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-agent antrea-cni
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-cni
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM antrea/base-ubi:${BUILD_TAG}
 

diff --git a/build/images/Dockerfile.build.agent.ubuntu b/build/images/Dockerfile.build.agent.ubuntu
@@ -18,18 +18,22 @@ FROM golang:${GO_VERSION} as antrea-build
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-agent antrea-cni
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-cni
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM antrea/base-ubuntu:${BUILD_TAG}
 

diff --git a/build/images/Dockerfile.build.controller.ubi b/build/images/Dockerfile.build.controller.ubi
@@ -39,18 +39,22 @@ ENV PATH /usr/local/go/bin:$PATH
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-controller
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-controller
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM ubuntu:22.04
 

diff --git a/build/images/Dockerfile.build.controller.ubuntu b/build/images/Dockerfile.build.controller.ubuntu
@@ -18,18 +18,22 @@ FROM golang:${GO_VERSION} as antrea-build
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-controller
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-controller
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM ubuntu:22.04
 

diff --git a/build/images/Dockerfile.build.ubi b/build/images/Dockerfile.build.ubi
@@ -39,18 +39,22 @@ ENV PATH /usr/local/go/bin:$PATH
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-agent antrea-controller antrea-cni
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-controller antrea-cni
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM antrea/base-ubi:${BUILD_TAG}
 

diff --git a/build/images/Dockerfile.build.ubuntu b/build/images/Dockerfile.build.ubuntu
@@ -18,18 +18,22 @@ FROM golang:${GO_VERSION} as antrea-build
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
-
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN make antrea-agent antrea-controller antrea-cni
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-controller antrea-cni
 # Disable CGO for antctl in case it is copied outside of the container image. It
 # also reduces the size of the binary and aligns with how we distribute antctl
 # in release assets.
-RUN CGO_ENABLED=0 make antctl-linux
-RUN mv bin/antctl-linux bin/antctl
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
 
 FROM antrea/base-ubuntu:${BUILD_TAG}