test(analyses-snapshot): add capability to run against local code (#1…

…6520)

.github/workflows/analyses-snapshot-test.yaml

@@ -45,12 +45,13 @@ jobs:
     timeout-minutes: 15
     runs-on: ubuntu-latest
     env:
+      BASE_IMAGE_NAME: opentrons-python-base:3.10
       ANALYSIS_REF: ${{ github.event.inputs.ANALYSIS_REF || github.head_ref || 'edge' }}
       SNAPSHOT_REF: ${{ github.event.inputs.SNAPSHOT_REF || github.head_ref || 'edge' }}
       # If we're running because of workflow_dispatch, use the user input to decide
       # whether to open a PR on failure. Otherwise, there is no user input,
       # so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr'
-      OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.events.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }}
+      OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }}
       PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'not a pr'}}
     steps:
       - name: Checkout Repository
@@ -71,9 +72,24 @@ jobs:
             echo "Analyses snapshots match ${{ env.PR_TARGET_BRANCH }} snapshots."
           fi
 
-      - name: Docker Build
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build base image
+        id: build_base_image
+        uses: docker/build-push-action@v6
+        with:
+          context: analyses-snapshot-testing/citools
+          file: analyses-snapshot-testing/citools/Dockerfile.base
+          push: false
+          load: true
+          tags: ${{ env.BASE_IMAGE_NAME }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build analysis image
         working-directory: analyses-snapshot-testing
-        run: make build-opentrons-analysis
+        run: make build-opentrons-analysis BASE_IMAGE_NAME=${{ env.BASE_IMAGE_NAME }} ANALYSIS_REF=${{ env.ANALYSIS_REF }} CACHEBUST=${{ github.run_number }}
 
       - name: Set up Python 3.13
         uses: actions/setup-python@v5
@@ -112,8 +128,8 @@ jobs:
           commit-message: 'fix(analyses-snapshot-testing): heal analyses snapshots'
           title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
           body: 'This PR was requested on the PR https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
-          branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF}}'
-          base: ${{ env.SNAPSHOT_REF}}
+          branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}'
+          base: ${{ env.SNAPSHOT_REF }}
 
       - name: Comment on feature PR
         if: always() && steps.create_pull_request.outcome == 'success' && github.event_name == 'pull_request'
@@ -135,5 +151,5 @@ jobs:
           commit-message: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
           title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
           body: 'The ${{ env.ANALYSIS_REF }} overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.'
-          branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF}}'
-          base: ${{ env.SNAPSHOT_REF}}
+          branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}'
+          base: ${{ env.SNAPSHOT_REF }}

analyses-snapshot-testing/Makefile

@@ -1,105 +1,121 @@
+BASE_IMAGE_NAME ?= opentrons-python-base:3.10
+CACHEBUST ?= $(shell date +%s)
+ANALYSIS_REF ?= edge
+PROTOCOL_NAMES ?= all
+OVERRIDE_PROTOCOL_NAMES ?= all
+OPENTRONS_VERSION ?= edge
+
+export OPENTRONS_VERSION # used for server
+export ANALYSIS_REF # used for analysis and snapshot test
+export PROTOCOL_NAMES # used for the snapshot test
+export OVERRIDE_PROTOCOL_NAMES # used for the snapshot test
+
+ifeq ($(CI), true)
+	PYTHON=python
+else
+	PYTHON=pyenv exec python
+endif
+
 .PHONY: black
 black:
-	python -m pipenv run python -m black .
+	$(PYTHON) -m pipenv run python -m black .
 
 .PHONY: black-check
 black-check:
-	python -m pipenv run python -m black . --check
+	$(PYTHON) -m pipenv run python -m black . --check
 
 .PHONY: ruff
 ruff:
-	python -m pipenv run python -m ruff check . --fix
+	$(PYTHON) -m pipenv run python -m ruff check . --fix
 
 .PHONY: ruff-check
 ruff-check:
-	python -m pipenv run python -m ruff check .
+	$(PYTHON) -m pipenv run python -m ruff check .
 
 .PHONY: mypy
 mypy:
-	python -m pipenv run python -m mypy automation tests citools
+	$(PYTHON) -m pipenv run python -m mypy automation tests citools
 
 .PHONY: lint
 lint: black-check ruff-check mypy
 
 .PHONY: format
 format: 
-	@echo runnning black
+	@echo "Running black"
 	$(MAKE) black
-	@echo running ruff
+	@echo "Running ruff"
 	$(MAKE) ruff
-	@echo formatting the readme with yarn prettier
+	@echo "Formatting the readme with yarn prettier"
 	$(MAKE) format-readme
 
 .PHONY: test-ci
 test-ci:
-	python -m pipenv run python -m pytest -m "emulated_alpha"
+	$(PYTHON) -m pipenv run python -m pytest -m "emulated_alpha"
 
 .PHONY: test-protocol-analysis
 test-protocol-analysis:
 	pipenv run python -m pytest -v tests/protocol_analyze_test.py
 
 .PHONY: setup
 setup: install-pipenv
-	python -m pipenv install
+	$(PYTHON) -m pipenv install
 
 .PHONY: teardown
 teardown:
-	python -m pipenv --rm
+	$(PYTHON) -m pipenv --rm
 
 .PHONY: format-readme
 format-readme:
-	yarn prettier --ignore-path .eslintignore --write analyses-snapshot-testing/**/*.md
+	yarn prettier --ignore-path .eslintignore --write analyses-snapshot-testing/**/*.md .github/workflows/analyses-snapshot-test.yaml
 
 .PHONY: install-pipenv
 install-pipenv:
-	python -m pip install -U pipenv
-
-ANALYSIS_REF ?= edge
-PROTOCOL_NAMES ?= all
-OVERRIDE_PROTOCOL_NAMES ?= all
-
-export ANALYSIS_REF
-export PROTOCOL_NAMES
-export OVERRIDE_PROTOCOL_NAMES
+	$(PYTHON) -m pip install -U pipenv
 
 .PHONY: snapshot-test
 snapshot-test:
 	@echo "ANALYSIS_REF is $(ANALYSIS_REF)"
 	@echo "PROTOCOL_NAMES is $(PROTOCOL_NAMES)"
 	@echo "OVERRIDE_PROTOCOL_NAMES is $(OVERRIDE_PROTOCOL_NAMES)"
-	python -m pipenv run pytest -k analyses_snapshot_test -vv
+	$(PYTHON) -m pipenv run pytest -k analyses_snapshot_test -vv
 
 .PHONY: snapshot-test-update
 snapshot-test-update:
 	@echo "ANALYSIS_REF is $(ANALYSIS_REF)"
 	@echo "PROTOCOL_NAMES is $(PROTOCOL_NAMES)"
 	@echo "OVERRIDE_PROTOCOL_NAMES is $(OVERRIDE_PROTOCOL_NAMES)"
-	python -m pipenv run pytest -k analyses_snapshot_test --snapshot-update
+	$(PYTHON) -m pipenv run pytest -k analyses_snapshot_test --snapshot-update
 
-CACHEBUST := $(shell date +%s)
+.PHONY: build-base-image
+build-base-image:
+	@echo "Building the base image $(BASE_IMAGE_NAME)"
+	docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) -f citools/Dockerfile.base -t $(BASE_IMAGE_NAME) citools/.
 
 .PHONY: build-opentrons-analysis
 build-opentrons-analysis:
 	@echo "Building docker image for $(ANALYSIS_REF)"
 	@echo "The image will be named opentrons-analysis:$(ANALYSIS_REF)"
 	@echo "If you want to build a different version, run 'make build-opentrons-analysis ANALYSIS_REF=<version>'"
-	@echo "Cache is always busted to ensure latest version of the code is used"
-	docker build --build-arg ANALYSIS_REF=$(ANALYSIS_REF) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-analysis:$(ANALYSIS_REF) citools/.
+	docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) --build-arg ANALYSIS_REF=$(ANALYSIS_REF) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-analysis:$(ANALYSIS_REF) -f citools/Dockerfile.analyze citools/.
+
+.PHONY: local-build
+local-build:
+	@echo "Building docker image for your local opentrons code"
+	@echo "The image will be named opentrons-analysis:local"
+	@echo "For a fresh build, run 'make local-build NO_CACHE=1'"
+	docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) $(BUILD_FLAGS) -t opentrons-analysis:local -f citools/Dockerfile.local .. || true
+	@echo "Build complete"
 
 .PHONY: generate-protocols
 generate-protocols:
-	python -m pipenv run python -m automation.data.protocol_registry
-
-
-OPENTRONS_VERSION ?= edge
-export OPENTRONS_VERSION
+	$(PYTHON) -m pipenv run python -m automation.data.protocol_registry
 
 .PHONY: build-rs
 build-rs:
 	@echo "Building docker image for opentrons-robot-server:$(OPENTRONS_VERSION)"
 	@echo "Cache is always busted to ensure latest version of the code is used"
 	@echo "If you want to build a different version, run 'make build-rs OPENTRONS_VERSION=chore_release-8.0.0'"
-	docker build --build-arg OPENTRONS_VERSION=$(OPENTRONS_VERSION) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-robot-server:$(OPENTRONS_VERSION) -f citools/Dockerfile.server .
+	docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) --build-arg OPENTRONS_VERSION=$(OPENTRONS_VERSION) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-robot-server:$(OPENTRONS_VERSION) -f citools/Dockerfile.server .
 
 .PHONY: run-flex
 run-flex:

analyses-snapshot-testing/README.md

@@ -18,7 +18,10 @@
 
 > This ALWAYS gets the remote code pushed to Opentrons/opentrons for the specified ANALYSIS_REF
 
-`make build-opentrons-analysis ANALYSIS_REF=chore_release-8.0.0`
+- build the base image
+  - `make build-base-image`
+- build the opentrons-analysis image
+  - `make build-opentrons-analysis ANALYSIS_REF=release`
 
 ## Running the tests locally
 
@@ -51,10 +54,28 @@
 
 ```shell
 cd analyses-snapshot-testing \
-&& make build-rs OPENTRONS_VERSION=chore_release-8.0.0 \
-&& make run-rs OPENTRONS_VERSION=chore_release-8.0.0`
+&& make build-base-image \
+&& make build-rs OPENTRONS_VERSION=release \
+&& make run-rs OPENTRONS_VERSION=release`
 ```
 
 ### Default OPENTRONS_VERSION=edge in the Makefile so you can omit it if you want latest edge
 
-`cd analyses-snapshot-testing && make build-rs && make run-rs`
+```shell
+cd analyses-snapshot-testing \
+&& make build-base-image \
+&& make build-rs \
+&& make run-rs
+```
+
+## Running the Analyses Battery against your local code
+
+> This copies in your local code to the container and runs the analyses battery against it.
+
+1. `make build-base-image`
+1. `make build-local`
+1. `make local-snapshot-test`
+
+You have the option to specify one or many protocols to run the analyses on. This is also described above [Running the tests against specific protocols](#running-the-tests-against-specific-protocols)
+
+- `make local-snapshot-test PROTOCOL_NAMES=Flex_S_v2_19_Illumina_DNA_PCR_Free OVERRIDE_PROTOCOL_NAMES=none`

analyses-snapshot-testing/citools/Dockerfile → analyses-snapshot-testing/citools/Dockerfile.analyze

@@ -1,10 +1,6 @@
-# Use 3.10 just like the app does
-FROM python:3.10-slim-bullseye
+ARG BASE_IMAGE_NAME=opentrons-python-base:3.10
 
-# Update packages and install git
-RUN apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install -y git libsystemd-dev
+FROM ${BASE_IMAGE_NAME}
 
 # Define build arguments
 ARG ANALYSIS_REF=edge

analyses-snapshot-testing/citools/Dockerfile.base

@@ -0,0 +1,7 @@
+# Use Python 3.10 as the base image
+FROM python:3.10-slim-bullseye
+
+# Update packages and install dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git libsystemd-dev build-essential pkg-config network-manager

analyses-snapshot-testing/citools/Dockerfile.local

@@ -0,0 +1,19 @@
+ARG BASE_IMAGE_NAME=opentrons-python-base:3.10
+
+FROM ${BASE_IMAGE_NAME}
+
+# Set the working directory in the container
+WORKDIR /opentrons
+
+# Copy everything from the build context into the /opentrons directory
+# root directory .dockerignore file is respected
+COPY . /opentrons
+
+# Install required packages from the copied code
+RUN python -m pip install -U ./shared-data/python
+RUN python -m pip install -U ./hardware[flex]
+RUN python -m pip install -U ./api
+RUN python -m pip install -U pandas==1.4.3
+
+# The default command to keep the container running
+CMD ["tail", "-f", "/dev/null"]

analyses-snapshot-testing/citools/Dockerfile.server

@@ -1,10 +1,6 @@
-# Use Python 3.10 as the base image
-FROM python:3.10-slim-bullseye
+ARG BASE_IMAGE_NAME=opentrons-python-base:3.10
 
-# Update packages and install dependencies
-RUN apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install -y git libsystemd-dev build-essential pkg-config network-manager
+FROM ${BASE_IMAGE_NAME}
 
 # Define build arguments
 ARG OPENTRONS_VERSION=edge

analyses-snapshot-testing/citools/generate_analyses.py

@@ -24,7 +24,7 @@
 HOST_RESULTS: Path = Path(Path(__file__).parent.parent, "analysis_results")
 ANALYSIS_SUFFIX: str = "analysis.json"
 ANALYSIS_TIMEOUT_SECONDS: int = 30
-ANALYSIS_CONTAINER_INSTANCES: int = 5
+MAX_ANALYSIS_CONTAINER_INSTANCES: int = 5
 
 console = Console()
 
@@ -241,6 +241,12 @@ def analyze_against_image(tag: str, protocols: List[TargetProtocol], num_contain
     return protocols
 
 
+def get_container_instances(protocol_len: int) -> int:
+    # Scaling linearly with the number of protocols
+    instances = max(1, min(MAX_ANALYSIS_CONTAINER_INSTANCES, protocol_len // 10))
+    return instances
+
+
 def generate_analyses_from_test(tag: str, protocols: List[Protocol]) -> None:
     """Generate analyses from the tests."""
     start_time = time.time()
@@ -260,6 +266,7 @@ def generate_analyses_from_test(tag: str, protocols: List[Protocol]) -> None:
                 protocol_custom_labware_paths_in_container(test_protocol),
             )
         )
-    analyze_against_image(tag, protocols_to_process, ANALYSIS_CONTAINER_INSTANCES)
+    instance_count = get_container_instances(len(protocols_to_process))
+    analyze_against_image(tag, protocols_to_process, instance_count)
     end_time = time.time()
     console.print(f"Clock time to generate analyses: {end_time - start_time:.2f} seconds.")