Revert "Fix Parallel Docker Platform Builds (#4013)" (#4018)

ethyca · Sep 1, 2023 · a339529 · a339529
1 parent 6a19f82
commit a339529
Show file tree

Hide file tree

Showing 4 changed files with 327 additions and 129 deletions.
diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml
@@ -12,14 +12,23 @@ env:
   DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
 
 jobs:
-  ParseTags:
+  push-fides:
     runs-on: ubuntu-latest
-    outputs:
-      prod_tag: ${{ steps.check-prod-tag.outputs.match }}
-      rc_tag: ${{ steps.check-rc-tag.outputs.match }}
-      alpha_tag: ${{ steps.check-alpha-tag.outputs.match }}
-      beta_tag: ${{ steps.check-beta-tag.outputs.match }}
     steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # This is required to properly tag images
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ env.DOCKER_USER }}
+          password: ${{ env.DOCKER_TOKEN }}
+
+      - name: Install Dev Requirements
+        run: pip install -r dev-requirements.txt
+
+
       - name: Check Prod Tag
         id: check-prod-tag
         run: |
@@ -28,7 +37,6 @@ jobs:
           else
             echo "match=false" >> $GITHUB_OUTPUT
           fi
-
       - name: Check RC Tag
         id: check-rc-tag
         run: |
@@ -37,7 +45,6 @@ jobs:
           else
             echo "match=false" >> $GITHUB_OUTPUT
           fi
-
       - name: Check alpha Tag
         id: check-alpha-tag
         run: |
@@ -46,7 +53,6 @@ jobs:
           else
             echo "match=false" >> $GITHUB_OUTPUT
           fi
-
       - name: Check beta Tag
         id: check-beta-tag
         run: |
@@ -56,61 +62,37 @@ jobs:
             echo "match=false" >> $GITHUB_OUTPUT
           fi
 
-      # if an RC git tag, also notify Fidesinfra to trigger a redeploy of rc env, to pick up our newly published images
-      - name: Send Repository Dispatch Event (RC redeploy)
-        if: steps.check-rc-tag.outputs.match == 'true'
-        uses: peter-evans/repository-dispatch@v2
-        with:
-          event-type: trigger-fidesinfra-deploy-fides-rc
-          repository: ethyca/fidesinfra
-          token: ${{ secrets.DISPATCH_ACCESS_TOKEN }}
-
-  Push:
-    runs-on: ubuntu-latest
-    needs: ParseTags
-    strategy:
-      # This matrix will effectively _try_ to run every permutation in parallel,
-      # skipping all of the tasks that don't match. This leaves a ton of "skipped" jobs
-      # but is the fastest way to get this working without overhauling the tag check logic.
-      matrix:
-        application: ["fides", "sample_app", "privacy_center"]
-        platform: ["x86", "ARM"]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is required to properly tag images
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ env.DOCKER_USER }}
-          password: ${{ env.DOCKER_TOKEN }}
-
-      - name: Install Dev Requirements
-        run: pip install -r dev-requirements.txt
-
       # if neither prod, rc, beta or alpha git tag, then push images with the ":dev" tag
       - name: Push Fides Dev Tag
-        if: needs.ParseTags.outputs.prod_tag == 'false' && needs.ParseTags.outputs.rc_tag == 'false' && needs.ParseTags.outputs.beta_tag == 'false' && needs.ParseTags.outputs.alpha_tag == 'false'
-        run: nox -s "push(${{ matrix.application }},dev,${{ matrix.platform }})"
+        if: steps.check-prod-tag.outputs.match == 'false' && steps.check-rc-tag.outputs.match == 'false' && steps.check-beta-tag.outputs.match == 'false' && steps.check-alpha-tag.outputs.match == 'false'
+        run: nox -s "push(dev)"
 
       # if a prod git tag, then we run the prod job to publish images tagged with the version number and a constant ":latest" tag
       - name: Push Fides Prod Tags
-        if: needs.ParseTags.outputs.prod_tag == 'true'
-        run: nox -s "push(${{ matrix.application }},prod,${{ matrix.platform }})"
+        if: steps.check-prod-tag.outputs.match == 'true'
+        run: nox -s "push(prod)"
 
       # if an RC git tag, then we run the rc job to publish images with an ":rc" tag
       - name: Push Fides RC Tags
-        if: needs.ParseTags.outputs.rc_tag == 'true'
-        run: nox -s "push(${{ matrix.application }},rc,${{ matrix.platform }})"
+        if: steps.check-rc-tag.outputs.match == 'true'
+        run: nox -s "push(rc)"
+
+      # if an RC git tag, also notify Fidesinfra to trigger a redeploy of rc env, to pick up our newly published images
+      - name: Send Repository Dispatch Event (RC redeploy)
+        if: steps.check-rc-tag.outputs.match == 'true'
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          event-type: trigger-fidesinfra-deploy-fides-rc
+          repository: ethyca/fidesinfra
+          token: ${{ secrets.DISPATCH_ACCESS_TOKEN }}
 
       # if an alpha or beta git tag, then we run the prerelease job to publish images with an ":prerelease" tag
       - name: Push Fides prerelease Tags
-        if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true'
-        run: nox -s "push(${{ matrix.application }},prerelease,${{ matrix.platform }})"
+        if: steps.check-alpha-tag.outputs.match == 'true' || steps.check-beta-tag.outputs.match == 'true'
+        run: nox -s "push(prerelease)"
 
       # if not a prod git tag, then we run the git-tag job to publish images with a git tag
       # if one exists on the current commit. the job is a no-op if the commit hasn't been tagged
       - name: Push Fides Commit Tags
-        if: needs.ParseTags.outputs.prod_tag == 'false'
-        run: nox -s "push(${{ matrix.application }},git-tag,${{ matrix.platform }})"
+        if: steps.check-prod-tag.outputs.match == 'false'
+        run: nox -s "push(git-tag)"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,9 +17,6 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.19.0...main)
 
-### Developer Experience
-- Fixed up parallelization in docker platform builds [#4013](https://github.com/ethyca/fides/pull/4013)
-
 ## [2.19.0](https://github.com/ethyca/fides/compare/2.18.0...2.19.0)
 
 ### Added

diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py
@@ -1,4 +1,6 @@
 """Contains the nox sessions for docker-related tasks."""
+from multiprocessing import Pool
+from subprocess import run
 from typing import Callable, Dict, List, Tuple
 
 import nox
@@ -16,6 +18,13 @@
 )
 from git_nox import get_current_tag, recognized_tag
 
+DOCKER_PLATFORMS = "linux/amd64,linux/arm64"
+
+
+def runner(args):
+    args_str = " ".join(args)
+    run(args_str, shell=True, check=True)
+
 
 def verify_git_tag(session: nox.Session) -> str:
     """
@@ -38,16 +47,13 @@ def verify_git_tag(session: nox.Session) -> str:
     return existing_commit_tag
 
 
-def generate_buildx_command(
-    image_tags: List[str],
-    docker_build_target: str,
-    platform: str,
-    dockerfile_path: str = ".",
+def generate_multiplatform_buildx_command(
+    image_tags: List[str], docker_build_target: str, dockerfile_path: str = "."
 ) -> Tuple[str, ...]:
     """
-    Generate the command for building and publishing an image.
+    Generate the command for building and publishing a multiplatform image.
 
-    See tests for example usage in `test_docker_nox.py`
+    See tests for example usage.
     """
     buildx_command: Tuple[str, ...] = (
         "docker",
@@ -56,7 +62,7 @@ def generate_buildx_command(
         "--push",
         f"--target={docker_build_target}",
         "--platform",
-        platform,
+        DOCKER_PLATFORMS,
         dockerfile_path,
     )
 
@@ -161,11 +167,41 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None:
     session.run(*build_command, external=True)
 
 
+def get_buildx_commands(tag_suffixes: List[str]) -> List[Tuple[str, ...]]:
+    """
+    Build and publish each image to Dockerhub
+    """
+    fides_tags = [f"{IMAGE}:{tag_suffix}" for tag_suffix in tag_suffixes]
+    fides_buildx_command = generate_multiplatform_buildx_command(
+        image_tags=fides_tags, docker_build_target="prod"
+    )
+
+    privacy_center_tags = [
+        f"{PRIVACY_CENTER_IMAGE}:{tag_suffix}" for tag_suffix in tag_suffixes
+    ]
+    privacy_center_buildx_command = generate_multiplatform_buildx_command(
+        image_tags=privacy_center_tags,
+        docker_build_target="prod_pc",
+    )
+
+    sample_app_tags = [
+        f"{SAMPLE_APP_IMAGE}:{tag_suffix}" for tag_suffix in tag_suffixes
+    ]
+    sample_app_buildx_command = generate_multiplatform_buildx_command(
+        image_tags=sample_app_tags,
+        docker_build_target="prod",
+        dockerfile_path="clients/sample-app",
+    )
+
+    buildx_commands = [
+        fides_buildx_command,
+        privacy_center_buildx_command,
+        sample_app_buildx_command,
+    ]
+    return buildx_commands
+
+
 @nox.session()
-@nox.parametrize(
-    "platform",
-    [nox.param("linux/amd64", id="x86"), nox.param("linux/arm64", id="ARM")],
-)
 @nox.parametrize(
     "tag",
     [
@@ -176,15 +212,7 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None:
         nox.param("git-tag", id="git-tag"),
     ],
 )
-@nox.parametrize(
-    "app",
-    [
-        nox.param("fides", id="fides"),
-        nox.param("privacy_center", id="privacy_center"),
-        nox.param("sample_app", id="sample_app"),
-    ],
-)
-def push(session: nox.Session, tag: str, app: str, platform: str) -> None:
+def push(session: nox.Session, tag: str) -> None:
     """
     Push the main image & extra apps to DockerHub:
       - ethyca/fides
@@ -199,9 +227,7 @@ def push(session: nox.Session, tag: str, app: str, platform: str) -> None:
     rc - Tags images with `rc` - used for rc tags
     git-tag - Tags images with the git tag of the current commit, if it exists
 
-    Example Calls:
-    nox -s "push(fides, prod, x86)"
-    nox -s "push(sample_app, prerelease, ARM)"
+    NOTE: This command also handles building images, including for multiple supported architectures.
     """
 
     # Create the buildx builder
@@ -226,34 +252,10 @@ def push(session: nox.Session, tag: str, app: str, platform: str) -> None:
         "prod": lambda: [get_current_tag(), "latest"],
     }
 
-    app_info_map = {
-        "fides": {"image": IMAGE, "target": "prod", "path": "."},
-        "privacy_center": {
-            "image": PRIVACY_CENTER_IMAGE,
-            "target": "prod_pc",
-            "path": ".",
-        },
-        "sample_app": {
-            "image": SAMPLE_APP_IMAGE,
-            "target": "prod",
-            "path": "clients/sample-app",
-        },
-    }
-    app_info: Dict[str, str] = app_info_map[app]
-
     # Get the list of Tupled commands to run
-    tag_suffixes: List[str] = param_tag_map[tag]()
-    full_tags: List[str] = [
-        f"{app_info['image']}:{tag_suffix}" for tag_suffix in tag_suffixes
-    ]
+    buildx_commands = get_buildx_commands(tag_suffixes=param_tag_map[tag]())
 
     # Parallel build the various images
-
-    buildx_command: Tuple[str, ...] = generate_buildx_command(
-        image_tags=full_tags,
-        docker_build_target=app_info["target"],
-        platform=platform,
-        dockerfile_path=app_info["path"],
-    )
-
-    session.run(*buildx_command, external=True)
+    number_of_processes = len(buildx_commands)
+    with Pool(number_of_processes) as process_pool:
+        process_pool.map(runner, buildx_commands)