Feat: Add kafka concurrent subscriber #1458

Workflow file for this run

.github/workflows/pr_tests.yaml at 23e62cf

	name: Run all tests

	on:
	schedule:
	- cron: "0 0 * * *"
	pull_request:
	types:
	- opened
	- synchronize
	- ready_for_review
	# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
	merge_group:
	types:
	- checks_requested

	env:
	ALL_PYTEST_MARKERS: "not nats and not kafka and not confluent and not rabbit and not redis"

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	pre-commit-check:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	env:
	SKIP: "docs"
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Set $PY environment variable
	run: echo "PY=$(python -VV \| sha256sum \| cut -d' ' -f1)" >> $GITHUB_ENV
	- uses: actions/cache@v4
	with:
	path: ~/.cache/pre-commit
	key: pre-commit\|${{ hashFiles('.pre-commit-config.yaml') }}
	- uses: pre-commit/[email protected]
	continue-on-error: true
	with:
	extra_args: --hook-stage manual --all-files
	- name: Commit
	uses: stefanzweifel/git-auto-commit-action@v5
	with:
	commit_message: "fix: add missing pre-commit changes"
	- uses: pre-commit/[email protected]

	test-basic:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
	pydantic-version: ["pydantic-v1", "pydantic-v2"]
	fail-fast: false

	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	- uses: actions/cache@v4
	id: cache
	with:
	path: ${{ env.pythonLocation }}
	key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-uv
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- name: Install Pydantic v1
	if: matrix.pydantic-version == 'pydantic-v1'
	run: uv pip install --system "pydantic>=1.10.0,<2.0.0"
	- name: Install Pydantic v2
	if: matrix.pydantic-version == 'pydantic-v2'
	run: uv pip install --system --pre "pydantic>=2.0.0b2,<3.0.0"
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh -vv
	-m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"
	env:
	COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
	CONTEXT: ${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	test-macos-latest:
	if: github.event.pull_request.draft == false
	runs-on: macos-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"

	test-windows-latest:
	if: github.event.pull_request.draft == false
	runs-on: windows-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"

	test-kafka-smoke:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[kafka,test-core,cli]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "not kafka"
	tests/brokers/kafka/test_test_client.py

	test-kafka-real:
	if: github.event.pull_request.draft == false
	needs:
	- test-basic
	- test-kafka-smoke
	runs-on: ubuntu-latest
	services:
	kafka:
	image: bitnami/kafka:3.5.0
	ports:
	- 9092:9092
	env:
	KAFKA_ENABLE_KRAFT: "true"
	KAFKA_CFG_NODE_ID: "1"
	KAFKA_CFG_PROCESS_ROLES: "broker,controller"
	KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
	KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
	KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
	KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092"
	KAFKA_BROKER_ID: "1"
	KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
	ALLOW_PLAINTEXT_LISTENER: "true"
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and kafka) or kafka"
	env:
	COVERAGE_FILE: coverage/.coverage.kafka-py
	CONTEXT: kafka-py
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.kafka-py
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	test-confluent-smoke:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[confluent,test-core,cli]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "not confluent"
	tests/brokers/confluent/test_test_client.py

	test-confluent-real:
	if: github.event.pull_request.draft == false
	needs:
	- test-basic
	- test-confluent-smoke
	runs-on: ubuntu-latest
	services:
	kafka:
	image: bitnami/kafka:3.5.0
	ports:
	- 9092:9092
	env:
	KAFKA_ENABLE_KRAFT: "true"
	KAFKA_CFG_NODE_ID: "1"
	KAFKA_CFG_PROCESS_ROLES: "broker,controller"
	KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
	KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
	KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
	KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092"
	KAFKA_BROKER_ID: "1"
	KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
	ALLOW_PLAINTEXT_LISTENER: "true"
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh -vv
	-m "(slow and confluent) or confluent"
	env:
	COVERAGE_FILE: coverage/.coverage.confluent-py
	CONTEXT: confluent-py
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.confluent-py
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	test-rabbit-smoke:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[rabbit,test-core,cli]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "not rabbit"
	tests/brokers/rabbit/test_test_client.py

	test-rabbit-real:
	if: github.event.pull_request.draft == false
	needs:
	- test-basic
	- test-rabbit-smoke
	runs-on: ubuntu-latest
	services:
	rabbitmq:
	image: rabbitmq:alpine
	ports:
	- 5672:5672
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and rabbit) or rabbit"
	env:
	COVERAGE_FILE: coverage/.coverage.rabbit-py
	CONTEXT: rabbit-py
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.rabbit-py
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	test-nats-smoke:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[nats,test-core,cli]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "not nats"
	tests/brokers/nats/test_test_client.py

	test-nats-real:
	if: github.event.pull_request.draft == false
	needs:
	- test-basic
	- test-nats-smoke
	runs-on: ubuntu-latest
	services:
	nats:
	image: diementros/nats:js
	ports:
	- 4222:4222
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and nats) or nats"
	env:
	COVERAGE_FILE: coverage/.coverage.nats-py
	CONTEXT: nats-py
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.nats-py
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	test-redis-smoke:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[redis,test-core,cli]
	- name: Test
	run: >
	bash scripts/test.sh
	-m "not redis"
	tests/brokers/redis/test_test_client.py

	test-redis-real:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	needs:
	- test-basic
	- test-redis-smoke
	services:
	nats:
	image: redis:alpine
	ports:
	- 6379:6379
	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"
	- name: Install Dependencies
	if: steps.cache.outputs.cache-hit != 'true'
	run: \|
	uv pip install --system .[optionals,testing]
	- run: mkdir coverage
	- name: Test
	run: >
	bash scripts/test.sh
	-m "(slow and redis) or redis"
	env:
	COVERAGE_FILE: coverage/.coverage.redis-py
	CONTEXT: redis-py
	- name: Store coverage files
	uses: actions/upload-artifact@v4
	with:
	name: .coverage.redis-py
	path: coverage
	if-no-files-found: error
	include-hidden-files: true

	coverage-combine:
	if: github.event.pull_request.draft == false
	needs:
	- test-basic
	- test-kafka-real
	- test-confluent-real
	- test-rabbit-real
	- test-nats-real
	- test-redis-real
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4
	- uses: astral-sh/setup-uv@v3
	with:
	version: "latest"
	- uses: actions/setup-python@v5
	with:
	python-version: "3.8"

	- name: Get coverage files
	uses: actions/download-artifact@v4
	with:
	pattern: .coverage*
	path: coverage
	merge-multiple: true

	- run: \|
	uv pip install --system coverage[toml]

	- run: ls -la coverage
	- run: coverage combine coverage
	- run: coverage report
	- run: coverage html --show-contexts --title "FastStream coverage for ${{ github.sha }}"

	- name: Store coverage html
	uses: actions/upload-artifact@v4
	with:
	name: coverage-html
	path: htmlcov

	# https://github.com/marketplace/actions/alls-green#why
	check: # This job does nothing and is only used for the branch protection
	# from: https://github.com/re-actors/alls-green
	# Important: For this to work properly, it is a must to have the job always
	# run, otherwise GitHub will make it skipped when any of the dependencies
	# fail. In some contexts, skipped is interpreted as success which may lead
	# to undesired, unobvious and even dangerous (as in security breach
	# "dangerous") side-effects.
	if: always()

	needs:
	- pre-commit-check
	- coverage-combine
	- test-macos-latest
	- test-windows-latest

	runs-on: ubuntu-latest

	steps:
	- name: Decide whether the needed jobs succeeded or failed
	uses: re-actors/alls-green@release/v1 # nosemgrep
	with:
	jobs: ${{ toJSON(needs) }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Feat: Add kafka concurrent subscriber #1458

Workflow file

Feat: Add kafka concurrent subscriber #1458

Jobs

Run details

Workflow file for this run