diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..4b3ab7d --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,229 @@ +name: Tests + +on: + push: + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + blazingmq-dependency: + name: Build BlazingMQ as a dependency + runs-on: ubuntu-latest + outputs: + blazingmq_sha: ${{ steps.get-sha.outputs.blazingmq_sha }} + steps: + - uses: actions/checkout@v4 + + - name: Checkout BlazingMQ + run: git clone --depth 1 https://github.com/bloomberg/blazingmq + + - name: Get latest BlazingMQ commit SHA + id: get-sha + working-directory: blazingmq + run: echo "blazingmq_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Try to get cached BlazingMQ build artifacts + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: blazingmq_artifacts.tar.gz + key: ${{ steps.get-sha.outputs.blazingmq_sha }} + + - name: Set up dependencies + if: steps.cache-restore.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -qy build-essential \ + gdb \ + curl \ + python3.10 \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libz-dev + + - name: Create install directory for BlazingMQ and its dependencies + if: steps.cache-restore.outputs.cache-hit != 'true' + run: mkdir -p blazingmq_artifacts + + - name: Skip building bison and google-benchmark + if: steps.cache-restore.outputs.cache-hit != 'true' + run: | + mkdir -p thirdparty/{bison,google-benchmark} + mkdir -p build/{bison,google-benchmark} + touch thirdparty/bison/.complete + touch build/google-benchmark/.complete + + - name: Build and install BlazingMQ and its dependencies + if: steps.cache-restore.outputs.cache-hit != 'true' + env: + DIR_INSTALL: 'blazingmq_artifacts' + run: /bin/bash bin/build-manylinux.sh + + - name: Save built BlazingMQ build artifacts + if: steps.cache-restore.outputs.cache-hit != 'true' + run: tar czf blazingmq_artifacts.tar.gz blazingmq_artifacts + + - name: Cache built BlazingMQ build artifacts + id: cache-save + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: blazingmq_artifacts.tar.gz + key: ${{ steps.get-sha.outputs.blazingmq_sha }} + + linux-check: + name: Test on Linux + needs: blazingmq-dependency + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + steps: + - uses: actions/checkout@v4 + - name: Try to get cached BlazingMQ build artifacts + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: blazingmq_artifacts.tar.gz + key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + - name: Restore cached BlazingMQ build artifacts + run: tar xzf blazingmq_artifacts.tar.gz + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Create virtual environment + run: | + python3 -m venv venv + - name: Install Python dependencies + run: | + ./venv/bin/python -m pip install --upgrade pip + ./venv/bin/python -m pip install -r requirements-dev.txt + - name: Install package manager dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential \ + gdb \ + curl \ + cmake \ + cmake \ + ninja-build \ + pkg-config \ + bison \ + libfl-dev \ + libbenchmark-dev \ + libz-dev + - name: Run tests + env: + BMQ_BROKER_URI: tcp://localhost:30114 + PREFIX: blazingmq_artifacts + PYTHON: ./venv/bin/python + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig + run: | + mkdir -p bmq/logs + mkdir -p bmq/storage/archive + ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config & + (sleep 5; make test-build && make test-install && make check) + + lint-docs: + name: Lint and Docs + needs: blazingmq-dependency + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Try to get cached BlazingMQ build artifacts + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: blazingmq_artifacts.tar.gz + key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + - name: Restore cached BlazingMQ build artifacts + run: tar xzf blazingmq_artifacts.tar.gz + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Create virtual environment + run: | + python3 -m venv venv + - name: Install Python dependencies + run: | + ./venv/bin/python -m pip install --upgrade pip + ./venv/bin/python -m pip install -r requirements-dev.txt + - name: Set up dependencies + run: + sudo apt-get install clang-format + - name: Install Package + run: | + ./venv/bin/python -m pip install -e . + env: + PREFIX: blazingmq_artifacts + PYTHON: ./venv/bin/python + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig + - name: Lint sources + env: + PYTHON: ./venv/bin/python + run: | + make lint + - name: Build docs + run: | + ./venv/bin/python -m towncrier build --version 99.99 --name blazingmq --keep + make docs + env: + PYTHON: ./venv/bin/python + SPHINXBUILD: ../venv/bin/sphinx-build + + coverage: + name: Coverage + needs: blazingmq-dependency + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Try to get cached BlazingMQ build artifacts + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: blazingmq_artifacts.tar.gz + key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + - name: Restore cached BlazingMQ build artifacts + run: tar xzf blazingmq_artifacts.tar.gz + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Create virtual environment + run: | + python3 -m venv venv + - name: Install Python dependencies + run: | + ./venv/bin/python3 -m pip install --upgrade pip + ./venv/bin/python3 -m pip install -r requirements-dev.txt + - name: Run tests with coverage + env: + BMQ_BROKER_URI: tcp://localhost:30114 + PREFIX: blazingmq_artifacts + PYTHON: ./venv/bin/python + PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig + run: | + mkdir -p bmq/logs + mkdir -p bmq/storage/archive + ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config & + (sleep 5; make coverage-install && make coverage) + - name: Output code coverage summary + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: coverage*.xml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7d2ab16..b5e2d7e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -2,10 +2,6 @@ name: Wheels on: push: - pull_request: - paths-ignore: - - "docs/**" - - "CONTRIBUTING.md" release: types: - published @@ -25,6 +21,8 @@ jobs: - uses: actions/checkout@v3 - name: Build sdist + env: + BLAZINGMQ_PYTHON_NO_PKGCONFIG: ’1’ run: pipx run build --sdist - uses: actions/upload-artifact@v3 @@ -37,14 +35,17 @@ jobs: name: tests path: tests + - uses: actions/upload-artifact@v3 + with: + name: bin + path: bin + choose_architectures: name: Decide which architectures to build wheels for runs-on: ubuntu-latest steps: - id: x86_64 run: echo "cibw_arch=x86_64" >> $GITHUB_OUTPUT - - id: i686 - run: echo "cibw_arch=i686" >> $GITHUB_OUTPUT - id: aarch64 if: github.event_name == 'release' && github.event.action == 'published' run: echo "cibw_arch=aarch64" >> $GITHUB_OUTPUT @@ -72,6 +73,10 @@ jobs: with: name: tests path: tests + - uses: actions/download-artifact@v3 + with: + name: bin + path: bin - uses: docker/setup-qemu-action@v2 if: runner.os == 'Linux' name: Set up QEMU @@ -84,12 +89,11 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.15.0 env: - CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} + CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_PRERELEASE_PYTHONS: True - CIBW_TEST_EXTRAS: test - CIBW_TEST_COMMAND: python -m pytest {package}/tests - CIBW_TEST_SKIP: "*aarch64*" + CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit + CIBW_TEST_REQUIRES: pytest mock pkgconfig - uses: actions/upload-artifact@v3 with: name: dist @@ -102,8 +106,8 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-11] - cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] + os: [macos-12] + cibw_python: ["cp311-*"] cibw_arch: ["x86_64", "arm64"] steps: @@ -115,22 +119,26 @@ jobs: with: name: tests path: tests + - uses: actions/download-artifact@v3 + with: + name: bin + path: bin - name: Extract sdist run: | tar zxvf dist/*.tar.gz --strip-components=1 - name: Sets env vars for compilation if: matrix.cibw_arch == 'arm64' run: | - echo "CFLAGS=-target arm64-apple-macos11" >> $GITHUB_ENV + echo "CFLAGS=-target arm64-apple-macos12" >> $GITHUB_ENV - name: Build wheels uses: pypa/cibuildwheel@v2.15.0 env: - CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }} - CIBW_PRERELEASE_PYTHONS: True - CIBW_TEST_EXTRAS: test - CIBW_TEST_COMMAND: pytest {package}/tests + CIBW_BUILD: ${{ matrix.cibw_python }} CIBW_BUILD_VERBOSITY: 1 + CIBW_PRERELEASE_PYTHONS: True + CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit + CIBW_TEST_REQUIRES: pytest mock pkgconfig MACOSX_DEPLOYMENT_TARGET: "11.0" - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/dco-check.yaml b/.github/workflows/dco-check.yaml new file mode 100644 index 0000000..286927e --- /dev/null +++ b/.github/workflows/dco-check.yaml @@ -0,0 +1,24 @@ +name: DCO check +on: + pull_request: + types: + - "opened" + - "reopened" + - "synchronize" + - "labeled" + - "unlabeled" + +jobs: + commits_check_job: + runs-on: ubuntu-latest + name: Commits Check + steps: + - name: Get PR Commits + id: "get-pr-commits" + uses: tim-actions/get-pr-commits@master + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: DCO Check + uses: tim-actions/dco@master + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/news-check.yaml b/.github/workflows/news-check.yaml new file mode 100644 index 0000000..f7212ce --- /dev/null +++ b/.github/workflows/news-check.yaml @@ -0,0 +1,23 @@ +name: News entry check +on: + pull_request: + paths: + - "src/**" + types: + - "opened" + - "reopened" + - "synchronize" + - "labeled" + - "unlabeled" + +jobs: + news_entry_check: + runs-on: ubuntu-latest + name: Check for news entry + steps: + - name: "Check for news entry" + uses: brettcannon/check-for-changed-files@v1 + with: + file-pattern: "news/*.rst" + skip-label: "skip news" + failure-message: "Missing a news file in ${file-pattern}; please add one or apply the ${skip-label} label to the pull request" diff --git a/MANIFEST.in b/MANIFEST.in index f8c70e9..7dd2e88 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,5 @@ recursive-exclude src/blazingmq *.cpp recursive-include src/blazingmq *.py recursive-include src/blazingmq *.pyx *.pxd recursive-include src/blazingmq *.vers +recursive-include src/blazingmq *.exp recursive-include src/declarations *.pxd diff --git a/bin/build-macos-universal.sh b/bin/build-macos-universal.sh index af72e31..618dd9f 100644 --- a/bin/build-macos-universal.sh +++ b/bin/build-macos-universal.sh @@ -105,6 +105,7 @@ if [ ! -e "${DIR_BUILD}/blazingmq/.complete" ]; then -DBDE_BUILD_TARGET_64=1 \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DINSTALL_TARGETS="bmqbrkr;bmq;mwc" \ -DCMAKE_INSTALL_LIBDIR="lib" \ -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ -DCMAKE_MODULE_PATH="${DIR_ROOT}" \ diff --git a/bin/build-manylinux.sh b/bin/build-manylinux.sh index 83989ff..84825a1 100644 --- a/bin/build-manylinux.sh +++ b/bin/build-manylinux.sh @@ -66,11 +66,13 @@ if [ ! -e "${DIR_THIRDPARTY}/cmake/.complete" ]; then mkdir -p "${DIR_THIRDPARTY}/cmake" pushd "${DIR_THIRDPARTY}/cmake" curl -s -L -O https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.sh - /bin/bash cmake-3.27.1-linux-x86_64.sh -- --skip-license --prefix="${DIR_INSTALL}" + /bin/bash cmake-3.27.1-linux-x86_64.sh -- --skip-license --exclude-subdir --prefix="${DIR_THIRDPARTY}/cmake" popd touch "${DIR_THIRDPARTY}/cmake/.complete" fi +PATH="${DIR_THIRDPARTY}/cmake/bin:$PATH" + if [ ! -e "${DIR_THIRDPARTY}/bison/.complete" ]; then # Build and install bison pushd "${DIR_THIRDPARTY}/bison" @@ -129,17 +131,17 @@ if [ ! -e "${DIR_BUILD}/blazingmq/.complete" ]; then -DBDE_BUILD_TARGET_64=1 \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DINSTALL_TARGETS="bmqbrkr;bmq;mwc" \ -DCMAKE_INSTALL_LIBDIR="lib64" \ -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ -DCMAKE_MODULE_PATH="${DIR_ROOT}" \ -DCMAKE_PREFIX_PATH="${DIR_THIRDPARTY}/bde-tools" \ -DCMAKE_TOOLCHAIN_FILE="${DIR_THIRDPARTY}/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake" \ -G "Ninja") - PKG_CONFIG_PATH="/usr/lib64/pkgconfig:${DIR_INSTALL}/lib64/pkgconfig" \ + PKG_CONFIG_PATH="$(pkg-config --variable pc_path pkg-config):${DIR_INSTALL}/lib64/pkgconfig" \ cmake -B "${DIR_BUILD}/blazingmq" -S "." "${CMAKE_OPTIONS[@]}" - cmake --build "${DIR_BUILD}/blazingmq" -j 16 --target bmq - cmake --install "${DIR_BUILD}/blazingmq" --component mwc-all - cmake --install "${DIR_BUILD}/blazingmq" --component bmq-all + cmake --build "${DIR_BUILD}/blazingmq" -j 16 --target all + cmake --install "${DIR_BUILD}/blazingmq" popd touch "${DIR_BUILD}/blazingmq/.complete" fi diff --git a/docs/Makefile b/docs/Makefile index b8bb1cb..837099c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD ?= sphinx-build SPHINXPROJ = blazingmq SOURCEDIR = . BUILDDIR = _build diff --git a/docs/conf.py b/docs/conf.py index 908e5af..69e98e9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -176,7 +176,8 @@ intersphinx_mapping = { "python": ( - "https://docs.python.org/3" + "https://docs.python.org/3", + None ) } diff --git a/pyproject.toml b/pyproject.toml index ce75532..279e48f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ directory = "news" [tool.cibuildwheel] build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*"] -skip = "*musllinux*{i686,aarch64}*" +skip = "*-musllinux_*" manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" diff --git a/requirements-test.txt b/requirements-test.txt index 93253de..7fd0094 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ mock pytest +pkgconfig diff --git a/setup.py b/setup.py index 1033ae8..dad5b09 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,20 @@ IS_MAC = platform.system() == "Darwin" IS_LINUX = platform.system() == "Linux" +# Making an sdist in principle does not require BDE+NTF+BlazingMQ. However, +# every time setup.py is run, it calls create_extension, which searches for bmq +# with pkgconfig. This is generally good; if we're building the SDK of course +# we need BlazingMQ built and installed on the system. For building an sdist, +# though, this is not needed. +# +# On GitHub CI, the sdist is generated in a separate container, before we run +# build_manylinux.sh to build the wheels in another container. We could run +# build_manylinux.sh prior to making the sdist, but that's a 30-minute +# waste. Instead, with this environment variable, we can selectively tell +# setup.py to not look for pkgconfig dependencies. This is only safe when +# making sdists, but that is exactly where we want it to happen. +SHOULD_SKIP_PKGCONFIG = "BLAZINGMQ_PYTHON_NO_PKGCONFIG" in os.environ + TEST_BUILD = False if "--test-build" in sys.argv: TEST_BUILD = True @@ -109,7 +123,11 @@ def create_extension(name, libraries, **kwargs): ext.extra_compile_args = extra_compile_args + list(ext.extra_compile_args) ext.extra_link_args = extra_link_args + list(ext.extra_link_args) - pkg_config = pkgconfig.parse(" ".join(libraries), static=True) + if SHOULD_SKIP_PKGCONFIG: + pkg_config_libraries = "" + else: + pkg_config_libraries = " ".join(libraries) + pkg_config = pkgconfig.parse(pkg_config_libraries, static=True) ext.define_macros = list(ext.define_macros) + pkg_config["define_macros"] ext.include_dirs = list(ext.include_dirs) + pkg_config["include_dirs"] ext.library_dirs = list(ext.library_dirs) + pkg_config["library_dirs"] diff --git a/src/blazingmq/_session.py b/src/blazingmq/_session.py index 26e89ba..aed3619 100644 --- a/src/blazingmq/_session.py +++ b/src/blazingmq/_session.py @@ -47,7 +47,7 @@ def __repr__(self) -> str: return "..." -def DefaultMonitor(): +def DefaultMonitor() -> Union[BasicHealthMonitor, None]: return None diff --git a/tests/broker-config/bmqbrkrcfg.json b/tests/broker-config/bmqbrkrcfg.json new file mode 100644 index 0000000..45fea6e --- /dev/null +++ b/tests/broker-config/bmqbrkrcfg.json @@ -0,0 +1,99 @@ +{ + "taskConfig": { + "allocatorType": "STACKTRACETEST", + "allocationLimit": 34359738368, + "logController": { + "fileName": "./bmq/logs/logs.%T.%p", + "fileMaxAgeDays": 10, + "rotationBytes": 268435456, + "logfileFormat": "%d (%t) %s %F:%l %m\n\n", + "consoleFormat": "%d (%t) %s %F:%l %m\n", + "loggingVerbosity": "INFO", + "consoleSeverityThreshold": "INFO", + "categories": [ + "BMQBRKR:INFO:green", + "BMQ*:INFO:green", + "MQB*:INFO:green", + "SIM*:INFO:gray", + "DMC*:INFO:yellow", + "BAEA.PERFORMANCEMONITOR:INFO:white" + ], + "syslog": { + "enabled": false, + "appName": "BMQ", + "logFormat": "%d (%t) %s %F:%l %m\n\n", + "verbosity": "" + } + } + }, + "appConfig": { + "brokerInstanceName": "default", + "brokerVersion": 999999, + "configVersion": 999999, + "etcDir": "", + "hostName": "earth", + "hostTags": "", + "hostDataCenter": "UNUSED", + "isRunningOnDev": false, + "logsObserverMaxSize": 1000, + "dispatcherConfig": { + "sessions": { + "numProcessors": 4, + "processorConfig": { + "queueSizeLowWatermark": 100000, + "queueSizeHighWatermark": 200000, + "queueSize": 500000 + } + }, + "queues": { + "numProcessors": 8, + "processorConfig": { + "queueSizeLowWatermark": 100000, + "queueSizeHighWatermark": 200000, + "queueSize": 500000 + } + }, + "clusters": { + "numProcessors": 4, + "processorConfig": { + "queueSizeLowWatermark": 100000, + "queueSizeHighWatermark": 200000, + "queueSize": 500000 + } + } + }, + "stats": { + "snapshotInterval": 1, + "printer": { + "printInterval": 60, + "file": "./bmq/logs/stat.%T.%p", + "maxAgeDays": 3, + "rotateBytes": 268435456, + "rotateDays": 1 + } + }, + "networkInterfaces": { + "heartbeats": { + "client": 0, + "downstreamBroker": 10, + "upstreamBroker": 10, + "clusterPeer": 10 + }, + "tcpInterface": { + "name": "TCPInterface", + "port": 30114, + "ioThreads": 4, + "maxConnections": 10000, + "lowWatermark": 4194304, + "highWatermark": 1073741824, + "nodeLowWatermark": 5242880, + "nodeHighWatermark": 10485760, + "heartbeatIntervalMs": 3000, + "useNtf": false + } + }, + "bmqconfConfig": { + "cacheTTLSeconds": 30 + } + } +} diff --git a/tests/broker-config/clusters.json b/tests/broker-config/clusters.json new file mode 100644 index 0000000..0db8ca2 --- /dev/null +++ b/tests/broker-config/clusters.json @@ -0,0 +1,89 @@ +{ + "myClusters": [ + { + "name": "local", + "clusterAttributes": { + "isCSLModeEnabled": true, + "isFSMWorkflow": false + }, + "nodes": [ + { + "id": 0, + "dataCenter": "UNSPECIFIED", + "name": "localhost", + "transport": { + "tcp": { + "endpoint": "tcp://localhost:30114" + } + } + } + ], + "partitionConfig": { + "name": "local", + "flushAtShutdown": true, + "location": "./bmq/storage", + "maxArchivedFileSets": 0, + "maxDataFileSize": 268435456, + "maxJournalFileSize": 67108864, + "maxQlistFileSize": 8388608, + "numPartitions": 1, + "preallocate": false, + "prefaultPages": false, + "archiveLocation": "./bmq/storage/archive", + "syncConfig": { + "fileChunkSize": 0, + "masterSyncMaxDurationMs": 0, + "maxAttemptsStorageSync": 0, + "partitionSyncDataReqTimeoutMs": 0, + "partitionSyncEventSize": 0, + "partitionSyncStateReqTimeoutMs": 0, + "startupRecoveryMaxDurationMs": 0, + "startupWaitDurationMs": 0, + "storageSyncReqTimeoutMs": 0 + } + }, + "masterAssignment": "E_LEADER_IS_MASTER_ALL", + "elector": { + "electionResultTimeoutMs": 4000, + "heartbeatBroadcastPeriodMs": 2000, + "heartbeatCheckPeriodMs": 1000, + "heartbeatMissCount": 10, + "initialWaitTimeoutMs": 8000, + "leaderSyncDelayMs": 80000, + "maxRandomWaitTimeoutMs": 3000, + "quorum": 0 + }, + "queueOperations": { + "ackWindowSize": 500, + "assignmentTimeoutMs": 15000, + "closeTimeoutMs": 300000, + "configureTimeoutMs": 300000, + "consumptionMonitorPeriodMs": 30000, + "keepaliveDurationMs": 1800000, + "openTimeoutMs": 300000, + "reopenMaxAttempts": 10, + "reopenRetryIntervalMs": 5000, + "reopenTimeoutMs": 43200000, + "shutdownTimeoutMs": 20000, + "stopTimeoutMs": 10000 + }, + "clusterMonitorConfig": { + "maxTimeLeader": 60, + "maxTimeMaster": 120, + "maxTimeNode": 120, + "maxTimeFailover": 240, + "thresholdLeader": 30, + "thresholdMaster": 60, + "thresholdNode": 60, + "thresholdFailover": 120 + }, + "messageThrottleConfig": { + "lowThreshold": 2, + "highThreshold": 4, + "lowInterval": 1000, + "highInterval": 3000 + } + } + ], + "proxyClusters": [] +} diff --git a/tests/broker-config/domains/bmq.test.mem.broadcast.json b/tests/broker-config/domains/bmq.test.mem.broadcast.json new file mode 100644 index 0000000..b8a90ab --- /dev/null +++ b/tests/broker-config/domains/bmq.test.mem.broadcast.json @@ -0,0 +1,37 @@ +{ + "definition": { + "location": "local", + "parameters": { + "maxDeliveryAttempts": 0, + "deduplicationTimeMs": 300000, + "consistency": { + "eventual": {} + }, + "storage": { + "config": { + "inMemory": {} + }, + "domainLimits": { + "bytes": 2097152, + "messages": 2000, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + }, + "queueLimits": { + "bytes": 1048576, + "messages": 1000, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + } + }, + "messageTtl": 300, + "maxProducers": 0, + "maxConsumers": 0, + "maxQueues": 0, + "maxIdleTime": 0, + "mode": { + "broadcast": {} + } + } + } +} diff --git a/tests/broker-config/domains/bmq.test.mmap.fanout.json b/tests/broker-config/domains/bmq.test.mmap.fanout.json new file mode 100644 index 0000000..c5bb23c --- /dev/null +++ b/tests/broker-config/domains/bmq.test.mmap.fanout.json @@ -0,0 +1,43 @@ +{ + "definition": { + "location": "local", + "parameters": { + "maxDeliveryAttempts": 0, + "deduplicationTimeMs": 300000, + "consistency": { + "eventual": {} + }, + "storage": { + "config": { + "fileBacked": {} + }, + "domainLimits": { + "bytes": 2097152, + "messages": 10000, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + }, + "queueLimits": { + "bytes": 1048576, + "messages": 500, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + } + }, + "messageTtl": 180, + "maxProducers": 0, + "maxConsumers": 0, + "maxQueues": 0, + "maxIdleTime": 0, + "mode": { + "fanout": { + "appIDs": [ + "foo", + "bar", + "baz" + ] + } + } + } + } +} diff --git a/tests/broker-config/domains/bmq.test.mmap.priority.json b/tests/broker-config/domains/bmq.test.mmap.priority.json new file mode 100644 index 0000000..2070b43 --- /dev/null +++ b/tests/broker-config/domains/bmq.test.mmap.priority.json @@ -0,0 +1,37 @@ +{ + "definition": { + "location": "local", + "parameters": { + "maxDeliveryAttempts": 0, + "deduplicationTimeMs": 300000, + "consistency": { + "eventual": {} + }, + "storage": { + "config": { + "fileBacked": {} + }, + "domainLimits": { + "bytes": 2097152, + "messages": 10000, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + }, + "queueLimits": { + "bytes": 1048576, + "messages": 10000, + "bytesWatermarkRatio": 0.8, + "messagesWatermarkRatio": 0.8 + } + }, + "messageTtl": 600, + "maxProducers": 0, + "maxConsumers": 0, + "maxQueues": 0, + "maxIdleTime": 0, + "mode": { + "priority": {} + } + } + } +} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bfd6f4d..ae6714f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,11 +15,12 @@ import uuid +import pytest + from blazingmq import BasicHealthMonitor from blazingmq import QueueOptions from blazingmq import Session from blazingmq.session_events import log_session_event -import pytest @pytest.fixture diff --git a/tests/integration/test_basic.py b/tests/integration/test_basic.py index fc11e72..035f672 100644 --- a/tests/integration/test_basic.py +++ b/tests/integration/test_basic.py @@ -20,11 +20,12 @@ import textwrap import weakref -from blazingmq import Session -from blazingmq import session_events import mock import pytest +from blazingmq import Session +from blazingmq import session_events + def test_on_session_events_processed(): # GIVEN diff --git a/tests/integration/test_confirm.py b/tests/integration/test_confirm.py index 2e9b377..2d8e06a 100644 --- a/tests/integration/test_confirm.py +++ b/tests/integration/test_confirm.py @@ -16,11 +16,12 @@ import queue import threading +import pytest + from blazingmq import Error from blazingmq import QueueOptions from blazingmq import Session from blazingmq.session_events import log_session_event -import pytest def test_confirmed_message_not_redelivered(unique_queue): diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py index 7ba8d26..030b5eb 100644 --- a/tests/integration/test_examples.py +++ b/tests/integration/test_examples.py @@ -20,11 +20,12 @@ import textwrap import time +import pytest + from blazingmq import AckStatus from blazingmq import QueueOptions from blazingmq import Session from blazingmq.session_events import log_session_event -import pytest def script_wrapper(queue, script): diff --git a/tests/integration/test_health_monitoring.py b/tests/integration/test_health_monitoring.py index 1c502ad..736fd42 100644 --- a/tests/integration/test_health_monitoring.py +++ b/tests/integration/test_health_monitoring.py @@ -15,14 +15,15 @@ import threading +import mock +import pytest + from blazingmq import BasicHealthMonitor from blazingmq import Error from blazingmq import QueueOptions from blazingmq import Session from blazingmq import session_events from blazingmq.testing import HostHealth -import mock -import pytest def test_receiving_host_health_events(): diff --git a/tests/integration/test_post.py b/tests/integration/test_post.py index d0914b1..89f2b60 100644 --- a/tests/integration/test_post.py +++ b/tests/integration/test_post.py @@ -17,13 +17,14 @@ import threading import weakref +import pytest + from blazingmq import AckStatus from blazingmq import CompressionAlgorithmType from blazingmq import QueueOptions from blazingmq import Session from blazingmq import exceptions from blazingmq.session_events import log_session_event -import pytest def test_post_success(default_session, unique_queue, zeroed_queue_options): diff --git a/tests/integration/test_queue.py b/tests/integration/test_queue.py index d3e8d9e..4de1159 100644 --- a/tests/integration/test_queue.py +++ b/tests/integration/test_queue.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + import blazingmq from blazingmq import exceptions -import pytest def test_bad_queue_uri(default_session, zeroed_queue_options): diff --git a/tests/unit/support.py b/tests/unit/support.py index 6c9e3e5..b271d65 100644 --- a/tests/unit/support.py +++ b/tests/unit/support.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock + from blazingmq import PropertyType from blazingmq import Session from blazingmq._ext import PROPERTY_TYPES_FROM_PY_MAPPING -import mock QUEUE_NAME = b"bmq://bmq.dummy_domain.some_namespace/dummy_queue" diff --git a/tests/unit/test_consumption.py b/tests/unit/test_consumption.py index 819233d..55bbd1b 100644 --- a/tests/unit/test_consumption.py +++ b/tests/unit/test_consumption.py @@ -15,11 +15,12 @@ import queue +import pytest + from blazingmq._enums import PropertyType from blazingmq._ext import Session from blazingmq._messages import AckStatus from blazingmq._messages import pretty_hex -import pytest from .support import BINARY from .support import BOOL diff --git a/tests/unit/test_ext_confirm.py b/tests/unit/test_ext_confirm.py index 1856366..ccb21da 100644 --- a/tests/unit/test_ext_confirm.py +++ b/tests/unit/test_ext_confirm.py @@ -15,10 +15,11 @@ import threading +import pytest + from blazingmq import exceptions from blazingmq._ext import Session from blazingmq._messages import create_message -import pytest from .support import QUEUE_NAME from .support import dummy_callback diff --git a/tests/unit/test_ext_post.py b/tests/unit/test_ext_post.py index b33d249..962613a 100644 --- a/tests/unit/test_ext_post.py +++ b/tests/unit/test_ext_post.py @@ -15,11 +15,12 @@ import weakref +import pytest + from blazingmq import CompressionAlgorithmType from blazingmq import exceptions from blazingmq._ext import COMPRESSION_ALGO_FROM_PY_MAPPING as compression_map from blazingmq._ext import Session -import pytest from .support import QUEUE_NAME from .support import dummy_callback diff --git a/tests/unit/test_ext_queue.py b/tests/unit/test_ext_queue.py index fbaae44..695672a 100644 --- a/tests/unit/test_ext_queue.py +++ b/tests/unit/test_ext_queue.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from blazingmq import exceptions from blazingmq._ext import Session -import pytest from .support import QUEUE_NAME from .support import dummy_callback diff --git a/tests/unit/test_ext_session.py b/tests/unit/test_ext_session.py index 9be407e..6bdd968 100644 --- a/tests/unit/test_ext_session.py +++ b/tests/unit/test_ext_session.py @@ -18,11 +18,12 @@ import sys import weakref +import pytest + from blazingmq import exceptions from blazingmq._ext import Session from blazingmq._ext import ensure_stop_session from blazingmq.session_events import InterfaceError -import pytest from .support import QUEUE_NAME from .support import dummy_callback diff --git a/tests/unit/test_ext_session_sending_properties.py b/tests/unit/test_ext_session_sending_properties.py index 58595c6..15526f4 100644 --- a/tests/unit/test_ext_session_sending_properties.py +++ b/tests/unit/test_ext_session_sending_properties.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from blazingmq import CompressionAlgorithmType from blazingmq._ext import COMPRESSION_ALGO_FROM_PY_MAPPING as compression_map from blazingmq._ext import Session -import pytest from .support import BINARY from .support import BOOL diff --git a/tests/unit/test_messages.py b/tests/unit/test_messages.py index e7f27e1..fcd0a2e 100644 --- a/tests/unit/test_messages.py +++ b/tests/unit/test_messages.py @@ -15,12 +15,13 @@ import weakref +import pytest + import blazingmq from blazingmq import _callbacks from blazingmq._messages import create_ack from blazingmq._messages import create_message from blazingmq._messages import create_message_handle -import pytest from .support import QUEUE_NAME from .support import mock diff --git a/tests/unit/test_queue_options.py b/tests/unit/test_queue_options.py index c881abb..633777c 100644 --- a/tests/unit/test_queue_options.py +++ b/tests/unit/test_queue_options.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import blazingmq import pytest +import blazingmq + def test_queue_options_repr(): # WHEN diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index cefbedc..bc39238 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -15,6 +15,8 @@ import re +import pytest + from blazingmq import BasicHealthMonitor from blazingmq import CompressionAlgorithmType from blazingmq import Error @@ -23,7 +25,6 @@ from blazingmq._messages import create_message from blazingmq._session import DEFAULT_TIMEOUT from blazingmq.testing import HostHealth -import pytest from .support import dummy_callback from .support import make_session diff --git a/tests/unit/test_session_events.py b/tests/unit/test_session_events.py index 616d516..b5e2693 100644 --- a/tests/unit/test_session_events.py +++ b/tests/unit/test_session_events.py @@ -15,10 +15,11 @@ import logging +import pytest + from blazingmq import _callbacks from blazingmq import _ext from blazingmq import session_events -import pytest from .support import mock diff --git a/tests/unit/test_session_sending_properties.py b/tests/unit/test_session_sending_properties.py index 01b85bc..97bcbe4 100644 --- a/tests/unit/test_session_sending_properties.py +++ b/tests/unit/test_session_sending_properties.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from blazingmq import Error from blazingmq import PropertyType -import pytest from .support import BINARY from .support import BOOL