Run CI #205

Workflow file for this run

	name: Run CI

	# Run this workflow every time a new commit pushed to your repository
	on:
	push:
	branches:
	- master
	- stable/*
	tags:
	- '*'
	pull_request:
	workflow_dispatch:
	schedule:
	- cron: '30 7 * * *'

	env:
	IMAGE_NAME: openformulieren/open-forms
	DJANGO_SETTINGS_MODULE: openforms.conf.ci
	HYPOTHESIS_PROFILE: ci
	CAMUNDA_API_BASE_URL: http://localhost:8080/engine-rest/
	CAMUNDA_USER: demo
	CAMUNDA_PASSWORD: demo

	jobs:

	tests:
	name: Run the Django test suite
	runs-on: ubuntu-latest

	services:
	postgres:
	image: postgres:14
	env:
	POSTGRES_HOST_AUTH_METHOD: trust
	ports:
	- 5432:5432
	# Needed because the postgres container does not provide a healthcheck
	options:
	--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
	--name postgres
	redis:
	image: redis:6
	ports:
	- 6379:6379
	steps:
	- uses: actions/checkout@v3
	- name: Set up backend environment
	uses: maykinmedia/[email protected]
	with:
	apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client gdal-bin'
	python-version: '3.10'
	optimize-postgres: 'yes'
	pg-service: 'postgres'
	setup-node: 'yes'
	npm-ci-flags: '--legacy-peer-deps'

	- name: Start CI docker services
	run: \|
	docker-compose -f docker-compose.ci.yml up -d
	docker-compose -f docker-compose.camunda.yml up -d
	working-directory: docker

	- name: Wait for Camunda to be up
	run: \|
	endpoint="${CAMUNDA_API_BASE_URL}version"
	version=""

	until [ $version ]; do
	echo "Checking if Camunda at ${CAMUNDA_API_BASE_URL} is up..."
	version=$(curl -u ${CAMUNDA_USER}:${CAMUNDA_PASSWORD} "$endpoint" -s \| jq -r ".version")
	sleep 2
	done

	echo "Running Camunda $version"

	- name: Run tests
	run: \|
	echo "# Profiling stats" >> $GITHUB_STEP_SUMMARY
	echo "" >> $GITHUB_OUTPUT

	python src/manage.py compilemessages
	python src/manage.py collectstatic --noinput --link
	coverage run \
	--concurrency=multiprocessing \
	--parallel-mode \
	src/manage.py test src \
	--parallel 2 \
	--exclude-tag=e2e \
	--verbosity 2
	coverage combine

	env:
	DJANGO_SETTINGS_MODULE: openforms.conf.ci
	SECRET_KEY: dummy
	DB_USER: postgres
	DB_PASSWORD: ''

	- name: Run JS tests
	run: npm test

	- name: Publish coverage report
	uses: codecov/codecov-action@v3

	- name: Generate OAS
	run: ./bin/generate_oas.sh openapi.yaml

	- name: Store generated OAS
	uses: actions/upload-artifact@v3
	with:
	name: open-forms-oas
	path: openapi.yaml
	retention-days: 1

	tests-reverse:
	name: Run the Django test suite in reverse
	runs-on: ubuntu-latest

	services:
	postgres:
	image: postgres:14
	env:
	POSTGRES_HOST_AUTH_METHOD: trust
	ports:
	- 5432:5432
	# Needed because the postgres container does not provide a healthcheck
	options:
	--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
	--name postgres
	redis:
	image: redis:6
	ports:
	- 6379:6379
	steps:
	- uses: actions/checkout@v3
	- name: Set up backend environment
	uses: maykinmedia/setup-django-backend@v1
	with:
	apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client gdal-bin'
	python-version: '3.10'
	optimize-postgres: 'yes'
	pg-service: 'postgres'
	setup-node: 'yes'
	npm-ci-flags: '--legacy-peer-deps'

	- name: Start CI docker services
	run: \|
	docker-compose -f docker-compose.ci.yml up -d
	docker-compose -f docker-compose.camunda.yml up -d
	working-directory: docker

	- name: Wait for Camunda to be up
	run: \|
	endpoint="${CAMUNDA_API_BASE_URL}version"
	version=""

	until [ $version ]; do
	echo "Checking if Camunda at ${CAMUNDA_API_BASE_URL} is up..."
	version=$(curl -u ${CAMUNDA_USER}:${CAMUNDA_PASSWORD} "$endpoint" -s \| jq -r ".version")
	sleep 2
	done

	echo "Running Camunda $version"

	- name: Run tests
	run: \|
	python src/manage.py compilemessages
	python src/manage.py collectstatic --noinput --link
	src/manage.py test src \
	--parallel 2 \
	--exclude-tag=e2e \
	--reverse

	env:
	DJANGO_SETTINGS_MODULE: openforms.conf.ci
	SECRET_KEY: dummy
	DB_USER: postgres
	DB_PASSWORD: ''

	e2etests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	browser:
	- chromium
	- firefox
	- webkit

	name: End-to-end tests, ${{ matrix.browser }}

	services:
	postgres:
	image: postgres:14
	env:
	POSTGRES_HOST_AUTH_METHOD: trust
	ports:
	- 5432:5432
	# Needed because the postgres container does not provide a healthcheck
	options:
	--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
	--name postgres
	redis:
	image: redis:6
	ports:
	- 6379:6379
	steps:
	- uses: actions/checkout@v3
	- name: Set up backend environment
	uses: maykinmedia/setup-django-backend@v1
	with:
	apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client gdal-bin'
	python-version: '3.10'
	optimize-postgres: 'yes'
	pg-service: 'postgres'
	setup-node: 'yes'
	npm-ci-flags: '--legacy-peer-deps'

	# See https://playwright.dev/python/docs/ci#caching-browsers
	- name: Cache Playwright browser
	id: cache-browser
	uses: actions/cache@v3
	with:
	path: /home/runner/.cache/ms-playwright
	key:
	${{ runner.os }}-${{ matrix.browser }}-playwright-${{ hashFiles('requirements/ci.txt') }}

	- name: Install playwright deps
	run: playwright install --with-deps ${{ matrix.browser }}

	- name: Run testsuite
	run: \|
	python src/manage.py compilemessages
	python src/manage.py collectstatic --noinput --link
	src/manage.py test src --tag=e2e
	env:
	DJANGO_SETTINGS_MODULE: openforms.conf.ci
	SECRET_KEY: dummy
	DB_USER: postgres
	DB_PASSWORD: ''
	E2E_DRIVER: ${{ matrix.browser }}
	# with 2FA enabled, for some reason this doesn't work on CI -> can't find
	# the inputs
	TWO_FACTOR_PATCH_ADMIN: 'no'

	docs:
	name: Build and check documentation
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v3
	with:
	path: open-forms

	- name: Set up backend environment
	uses: maykinmedia/setup-django-backend@v1
	with:
	apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gdal-bin'
	python-version: '3.10'
	setup-node: 'no'
	working-directory: 'open-forms'

	- name: Determine SDK version to checkout
	id: sdk-ref
	run: \|
	# Strip git ref prefix from version
	VERSION=$(echo "${{ github.ref }}" \| sed -e 's,./$.$,\1,')

	# Strip "v" prefix from tag name (if present at all)
	[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION \| sed -e 's/^v//')

	# default to version in .sdk-release file
	SDK_REF=$(cat .sdk-release \| tr -d '[:space:]')

	case $VERSION in
	# if building master -> include main of SDK
	master) SDK_REF=main;;
	# PRs result in version 'merge' that'll go to master -> include main of SDK
	merge) SDK_REF=main;;
	esac

	echo "sdk_ref=${SDK_REF}" >> $GITHUB_OUTPUT
	working-directory: open-forms

	- name: Checkout SDK repository
	uses: actions/checkout@v3
	with:
	repository: 'open-formulieren/open-forms-sdk'
	ref: ${{ steps.sdk-ref.outputs.sdk_ref }}
	path: open-forms-sdk

	- name: Setup symlinks
	run: \|
	ln -s $(pwd)/open-forms-sdk/CHANGELOG.rst open-forms/docs/changelog-sdk.rst

	- name: Build and test docs
	run: \|
	export OPENSSL_CONF=$(pwd)/openssl.conf
	pytest check_sphinx.py -v --tb=auto
	working-directory: open-forms/docs

	# see https://github.com/orgs/community/discussions/26671
	docker_build_setup:
	name: Set up docker build 'dynamic' env variables
	runs-on: ubuntu-latest
	outputs:
	image-name: ${{ steps.set-output-defaults.outputs.image-name }}
	steps:
	- name: Set output with default values
	id: set-output-defaults
	run: \|
	echo "image-name=${{ env.IMAGE_NAME }}" >> $GITHUB_OUTPUT

	docker_build:
	needs: docker_build_setup
	strategy:
	matrix:
	# KEEP IN SYNC WITH docker_push JOB
	target:
	- env: production
	extensions: ''
	image_tag_prefix: ''
	- env: extensions
	extensions: 'token_exchange,prefill_haalcentraalhr'
	image_tag_prefix: 'all-extensions-'
	uses: ./.github/workflows/build-image.yml
	with:
	image_name: ${{ needs.docker_build_setup.outputs.image-name }}
	image_tag_prefix: ${{ matrix.target.image_tag_prefix }}
	target_env: ${{ matrix.target.env }}
	extensions: ${{ matrix.target.extensions }}

	image_scan:
	runs-on: ubuntu-latest
	name: Scan docker image
	needs:
	- docker_build

	steps:
	# So the scanner gets commit meta-information
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Derive version
	id: vars
	run: \|
	# Strip git ref prefix from version
	VERSION=$(echo "${{ github.ref }}" \| sed -e 's,./$.$,\1,')
	# Strip "v" prefix from tag name (if present at all)
	[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION \| sed -e 's/^v//')
	# Use Docker `latest` tag convention
	[ "$VERSION" == "master" ] && VERSION=latest
	# PRs result in version 'merge' -> transform that into 'latest'
	[ "$VERSION" == "merge" ] && VERSION=latest
	echo "version=${VERSION}" >> $GITHUB_OUTPUT
	- name: Download built image
	uses: actions/download-artifact@v3
	with:
	name: docker-image-all-extensions-${{ steps.vars.outputs.version }}
	- name: Scan image with Trivy
	uses: aquasecurity/trivy-action@master
	with:
	input: /github/workspace/image.tar # from download-artifact
	format: 'sarif'
	output: 'trivy-results-docker.sarif'
	ignore-unfixed: true
	- name: Upload results to GH Security tab
	uses: github/codeql-action/upload-sarif@v2
	with:
	sarif_file: 'trivy-results-docker.sarif'

	oas-up-to-date:
	needs: tests
	name: Check for unexepected OAS changes
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Download generated OAS
	uses: actions/download-artifact@v3
	with:
	name: open-forms-oas
	- name: Check for OAS changes
	run: \|
	diff openapi.yaml src/openapi.yaml
	- name: Write failure markdown
	if: ${{ failure() }}
	run: \|
	echo 'Run the following command locally and commit the changes' >> $GITHUB_STEP_SUMMARY
	echo '' >> $GITHUB_STEP_SUMMARY
	echo '```bash' >> $GITHUB_STEP_SUMMARY
	echo './bin/generate_oas.sh' >> $GITHUB_STEP_SUMMARY
	echo '```' >> $GITHUB_STEP_SUMMARY

	oas-lint:
	needs: oas-up-to-date
	name: Validate OAS
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Download generated OAS
	uses: actions/download-artifact@v3
	with:
	name: open-forms-oas
	- name: Use Node.js
	uses: actions/setup-node@v3
	with:
	node-version-file: '.nvmrc'
	- name: Install spectral
	run: npm install -g @stoplight/[email protected]
	- name: Run OAS linter
	run: spectral lint ./openapi.yaml

	oas-postman:
	needs: oas-up-to-date
	name: Generate Postman collection from OAS
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Download generated OAS
	uses: actions/download-artifact@v3
	with:
	name: open-forms-oas
	- name: Use Node.js
	uses: actions/setup-node@v3
	with:
	node-version-file: '.nvmrc'
	- name: Install dependencies
	run: npm install -g openapi-to-postmanv2
	- name: Create tests folder
	run: mkdir -p ./tests/postman
	- name: Generate Postman collection
	run: openapi2postmanv2 -s ./openapi.yaml -o ./tests/postman/collection.json --pretty

	oas-generate-sdks:
	needs: oas-up-to-date
	name: Generate SDKs from OAS
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Download generated OAS
	uses: actions/download-artifact@v3
	with:
	name: open-forms-oas
	- name: Use Node.js
	uses: actions/setup-node@v3
	with:
	node-version-file: '.nvmrc'
	- name: Install dependencies
	run: npm install -g @openapitools/[email protected]
	- name: Validate schema
	run: openapi-generator-cli validate -i ./openapi.yaml
	- name: Set the version of openapi-generator which gets used
	run: \|
	openapi-generator-cli version-manager set 7.0.0
	- name: Generate Java client
	run:
	openapi-generator-cli generate -i ./openapi.yaml
	--global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o
	./sdks/java -g java
	--additional-properties=dateLibrary=java8,java8=true,optionalProjectFile=false,optionalAssemblyInfo=false
	- name: Generate .NET Full Framework client
	run:
	openapi-generator-cli generate -i ./openapi.yaml
	--global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o
	./sdks/net -g csharp
	--additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false
	- name: Generate Python client
	run:
	openapi-generator-cli generate -i ./openapi.yaml
	--global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o
	./sdks/python -g python
	--additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false+

	docker_push:
	needs:
	- tests
	- e2etests
	- docker_build
	- oas-lint
	- oas-postman
	- oas-generate-sdks

	name: Push Docker image
	runs-on: ubuntu-latest
	if: github.event_name == 'push' # Exclude PRs

	strategy:
	matrix:
	# KEEP IN SYNC WITH docker_build JOB
	target:
	- env: production
	image_tag_prefix: ''
	- env: extensions
	image_tag_prefix: 'all-extensions-'

	steps:
	- uses: actions/checkout@v3

	- name: Derive version
	id: vars
	run: \|
	# Strip git ref prefix from version
	VERSION=$(echo "${{ github.ref }}" \| sed -e 's,./$.$,\1,')

	# Strip "v" prefix from tag name (if present at all)
	[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION \| sed -e 's/^v//')

	# Use Docker `latest` tag convention
	[ "$VERSION" == "master" ] && VERSION=latest

	# PRs result in version 'merge' -> transform that into 'latest'
	[ "$VERSION" == "merge" ] && VERSION=latest

	echo "version=${VERSION}" >> $GITHUB_OUTPUT

	- name: Download built image
	uses: actions/download-artifact@v3
	with:
	name: docker-image-${{ matrix.target.image_tag_prefix }}${{ steps.vars.outputs.version }}

	- name: Load image
	run: \|
	docker image load -i image.tar

	- name: Log into registry
	run:
	echo "${{ secrets.DOCKER_TOKEN }}" \| docker login -u ${{ secrets.DOCKER_USERNAME }}
	--password-stdin

	- name: Push the Docker image (production)
	run: docker push $IMAGE_NAME:$TAG
	env:
	TAG: ${{ matrix.target.image_tag_prefix }}${{ steps.vars.outputs.version }}

	update-docker-readme:
	needs:
	- docker_build_setup
	- docker_push
	uses: ./.github/workflows/dockerhub-description.yml
	with:
	image_name: ${{ needs.docker_build_setup.outputs.image-name }}
	secrets:
	DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
	DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Run CI #205

Workflow file

Run CI #205

Jobs

Run details

Workflow file for this run