From f1ff2834e39a6af3d07e5a4a23d93d9b4edbe637 Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 09:03:38 -0500 Subject: [PATCH 01/30] Add github actions file (no kafka yet) --- .github/workflows/elixir.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/elixir.yml diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml new file mode 100644 index 00000000..e3b8020e --- /dev/null +++ b/.github/workflows/elixir.yml @@ -0,0 +1,41 @@ +name: Elixir CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + MIX_ENV: test + +jobs: + build: + + runs-on: ubuntu-18.04 + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + - name: Install Dependencies + run: mix deps.get + - name: Build Dependencies + run: mix deps.compile + - name: Compile + run: mix compile --warnings-as-errors + - name: Check format + run: mix format --check-formatted + - name: Credo + run: mix credo --strict + - name: Run Tests + run: mix test From a8cb67c02f9fe9906405d01dd9f7140bc19f6c0e Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 09:05:08 -0500 Subject: [PATCH 02/30] Remove .travis.yml --- .travis.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 594e03fb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: elixir -branches: - only: - - "master" -cache: - directories: - - deps - - _build -matrix: - include: - - elixir: 1.9.4 - otp_release: 22.3.3 - env: COVERALLS=true CREDO=true - - elixir: 1.8.2 - otp_release: 22.3.3 - - elixir: 1.7.4 - otp_release: 22.3.3 - - elixir: 1.6.6 - otp_release: 20.3 -services: - - docker -env: - global: - - COVERALLS=false - - CREDO=false -before_script: - - ./scripts/docker_up.sh -script: - - MIX_ENV=test mix deps.compile - - MIX_ENV=test mix compile --warnings-as-errors - - ./scripts/ci_tests.sh From 0e118b22b27c5a25214dae0df70400e2d383bca5 Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 09:07:43 -0500 Subject: [PATCH 03/30] Add docker-compose up --- .github/workflows/elixir.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index e3b8020e..069dce3c 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -22,6 +22,8 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh - name: Setup elixir uses: actions/setup-elixir@v1 with: @@ -33,9 +35,5 @@ jobs: run: mix deps.compile - name: Compile run: mix compile --warnings-as-errors - - name: Check format - run: mix format --check-formatted - - name: Credo - run: mix credo --strict - name: Run Tests run: mix test From e539d4de13f03223059575233e724a5ceaa851e8 Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 09:14:36 -0500 Subject: [PATCH 04/30] Run integration tests --- .github/workflows/elixir.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 069dce3c..dbf0a235 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -36,4 +36,4 @@ jobs: - name: Compile run: mix compile --warnings-as-errors - name: Run Tests - run: mix test + run: ./scripts/ci_tests.sh From 13fba983df451c900790fb4d9b3938e004624b6a Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 09:31:07 -0500 Subject: [PATCH 05/30] Docker ps --- .github/workflows/elixir.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index dbf0a235..ac43e94f 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -35,5 +35,7 @@ jobs: run: mix deps.compile - name: Compile run: mix compile --warnings-as-errors + - name: Docker ps + run: docker ps - name: Run Tests run: ./scripts/ci_tests.sh From c5e78b00fbe9963cbb599477134de293994c06c0 Mon Sep 17 00:00:00 2001 From: Dan Swain Date: Sat, 7 Nov 2020 10:06:26 -0500 Subject: [PATCH 06/30] Try services --- .github/workflows/elixir.yml | 84 ++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index ac43e94f..a4ca97e9 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -20,6 +20,90 @@ jobs: - elixir: 1.10.2 erlang: 22.3 + services: + zookeeper: + image: wurstmeister/zookeeper:3.4.6 + ports: + - "2181:2181" + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] + interval: 5s + timeout: 10s + retries: 15 + start_period: 10s + + kafka1: + hostname: kafka1 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9093:9093" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_PORT: 9093 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka1 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy + + kafka2: + hostname: kafka2 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9094:9094" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_PORT: 9094 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka2 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy + + kafka3: + hostname: kafka3 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9095:9095" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ADVERTISED_PORT: 9095 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka3 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy + steps: - uses: actions/checkout@v2 - name: Docker-compose up From e13cc3717c2554815c3118d994ef68eb2098c117 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:09:58 -0600 Subject: [PATCH 07/30] Fix indentation in Services; remove docker-compose up --- .github/workflows/elixir.yml | 146 +++++++++++++++++------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index a4ca97e9..c5598c76 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -32,82 +32,82 @@ jobs: retries: 15 start_period: 10s - kafka1: - hostname: kafka1 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9093:9093" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_PORT: 9093 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka1 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + kafka1: + hostname: kafka1 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9093:9093" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_PORT: 9093 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka1 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy - kafka2: - hostname: kafka2 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9094:9094" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_PORT: 9094 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka2 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + kafka2: + hostname: kafka2 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9094:9094" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_PORT: 9094 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka2 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy - kafka3: - hostname: kafka3 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9095:9095" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 3 - KAFKA_ADVERTISED_PORT: 9095 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka3 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + kafka3: + hostname: kafka3 + image: wurstmeister/kafka:0.11.0.1 + ports: + - "9095:9095" + volumes: + - ./ssl:/ssl + - ./scripts/kafka_check_health:/kafka_check_health + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ADVERTISED_PORT: 9095 + KAFKA_PORT: 9092 + KAFKA_HOST_NAME: kafka3 + healthcheck: + test: /kafka_check_health + interval: 10s + timeout: 10s + retries: 30 + start_period: 10s + depends_on: + zookeeper: + condition: service_healthy steps: - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh + # - name: Docker-compose up + # run: ./scripts/docker_up.sh - name: Setup elixir uses: actions/setup-elixir@v1 with: @@ -119,7 +119,7 @@ jobs: run: mix deps.compile - name: Compile run: mix compile --warnings-as-errors - - name: Docker ps - run: docker ps + # - name: Docker ps + # run: docker ps - name: Run Tests run: ./scripts/ci_tests.sh From 6cbb6cf10a52ab86784f5582e9695cd7040dcf86 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:13:21 -0600 Subject: [PATCH 08/30] Run CI on all branches --- .github/workflows/elixir.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index c5598c76..dc9a4caf 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -1,18 +1,18 @@ name: Elixir CI -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request] + # push: + # branches: [ master ] + # pull_request: + # branches: [ master ] -env: - MIX_ENV: test jobs: build: runs-on: ubuntu-18.04 + env: + MIX_ENV: test strategy: matrix: From f2af33cb344a1112ff9d542682626bd3a3f665f7 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:18:34 -0600 Subject: [PATCH 09/30] Remove duplicate depends_on in docker-compose.yml --- docker-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 700c59ac..58fb76ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,6 @@ services: image: wurstmeister/kafka:0.11.0.1 ports: - "9093:9093" - depends_on: - - zookeeper volumes: - ./ssl:/ssl - ./scripts/kafka_check_health:/kafka_check_health @@ -43,8 +41,6 @@ services: image: wurstmeister/kafka:0.11.0.1 ports: - "9094:9094" - depends_on: - - zookeeper volumes: - ./ssl:/ssl - ./scripts/kafka_check_health:/kafka_check_health @@ -69,8 +65,6 @@ services: image: wurstmeister/kafka:0.11.0.1 ports: - "9095:9095" - depends_on: - - zookeeper volumes: - ./ssl:/ssl - ./scripts/kafka_check_health:/kafka_check_health From daaa7f704939d7962f1bc4c3b88dae1fdf56bf47 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:20:53 -0600 Subject: [PATCH 10/30] Try a straight dockerized CI mimicking Travis --- .github/workflows/dockerized.yml | 122 +++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/workflows/dockerized.yml diff --git a/.github/workflows/dockerized.yml b/.github/workflows/dockerized.yml new file mode 100644 index 00000000..a627cd25 --- /dev/null +++ b/.github/workflows/dockerized.yml @@ -0,0 +1,122 @@ +name: Elixir Dockerized CI + +on: [push, pull_request] + # push: + # branches: [ master ] + # pull_request: + # branches: [ master ] + + +jobs: + build-dockerized: + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + # services: + # zookeeper: + # image: wurstmeister/zookeeper:3.4.6 + # ports: + # - "2181:2181" + # healthcheck: + # test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] + # interval: 5s + # timeout: 10s + # retries: 15 + # start_period: 10s + + # kafka1: + # image: wurstmeister/kafka:0.11.0.1 + # ports: + # - "9093:9093" + # volumes: + # - ./ssl:/ssl + # - ./scripts/kafka_check_health:/kafka_check_health + # env_file: docker-compose-kafka.env + # environment: + # KAFKA_BROKER_ID: 1 + # KAFKA_ADVERTISED_PORT: 9093 + # KAFKA_PORT: 9092 + # KAFKA_HOST_NAME: kafka1 + # healthcheck: + # test: /kafka_check_health + # interval: 10s + # timeout: 10s + # retries: 30 + # start_period: 10s + # depends_on: + # zookeeper: + # condition: service_healthy + + # kafka2: + # image: wurstmeister/kafka:0.11.0.1 + # ports: + # - "9094:9094" + # volumes: + # - ./ssl:/ssl + # - ./scripts/kafka_check_health:/kafka_check_health + # env_file: docker-compose-kafka.env + # environment: + # KAFKA_BROKER_ID: 2 + # KAFKA_ADVERTISED_PORT: 9094 + # KAFKA_PORT: 9092 + # KAFKA_HOST_NAME: kafka2 + # healthcheck: + # test: /kafka_check_health + # interval: 10s + # timeout: 10s + # retries: 30 + # start_period: 10s + # depends_on: + # zookeeper: + # condition: service_healthy + + # kafka3: + # image: wurstmeister/kafka:0.11.0.1 + # ports: + # - "9095:9095" + # volumes: + # - ./ssl:/ssl + # - ./scripts/kafka_check_health:/kafka_check_health + # env_file: docker-compose-kafka.env + # environment: + # KAFKA_BROKER_ID: 3 + # KAFKA_ADVERTISED_PORT: 9095 + # KAFKA_PORT: 9092 + # KAFKA_HOST_NAME: kafka3 + # healthcheck: + # test: /kafka_check_health + # interval: 10s + # timeout: 10s + # retries: 30 + # start_period: 10s + # depends_on: + # zookeeper: + # condition: service_healthy + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + - name: Install Dependencies + run: mix deps.get + - name: Build Dependencies + run: mix deps.compile + - name: Compile + run: mix compile --warnings-as-errors + - name: Docker ps + run: docker ps + - name: Run Tests + run: ./scripts/ci_tests.sh From 784f52a914e7fe5616f05ff631be582ed79404fe Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:51:18 -0600 Subject: [PATCH 11/30] create test.yml to run CI Test Suite --- .github/workflows/dockerized.yml | 122 ------------------------------ .github/workflows/elixir.yml | 125 ------------------------------- .github/workflows/test.yml | 40 ++++++++++ 3 files changed, 40 insertions(+), 247 deletions(-) delete mode 100644 .github/workflows/dockerized.yml delete mode 100644 .github/workflows/elixir.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/dockerized.yml b/.github/workflows/dockerized.yml deleted file mode 100644 index a627cd25..00000000 --- a/.github/workflows/dockerized.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Elixir Dockerized CI - -on: [push, pull_request] - # push: - # branches: [ master ] - # pull_request: - # branches: [ master ] - - -jobs: - build-dockerized: - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - # services: - # zookeeper: - # image: wurstmeister/zookeeper:3.4.6 - # ports: - # - "2181:2181" - # healthcheck: - # test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] - # interval: 5s - # timeout: 10s - # retries: 15 - # start_period: 10s - - # kafka1: - # image: wurstmeister/kafka:0.11.0.1 - # ports: - # - "9093:9093" - # volumes: - # - ./ssl:/ssl - # - ./scripts/kafka_check_health:/kafka_check_health - # env_file: docker-compose-kafka.env - # environment: - # KAFKA_BROKER_ID: 1 - # KAFKA_ADVERTISED_PORT: 9093 - # KAFKA_PORT: 9092 - # KAFKA_HOST_NAME: kafka1 - # healthcheck: - # test: /kafka_check_health - # interval: 10s - # timeout: 10s - # retries: 30 - # start_period: 10s - # depends_on: - # zookeeper: - # condition: service_healthy - - # kafka2: - # image: wurstmeister/kafka:0.11.0.1 - # ports: - # - "9094:9094" - # volumes: - # - ./ssl:/ssl - # - ./scripts/kafka_check_health:/kafka_check_health - # env_file: docker-compose-kafka.env - # environment: - # KAFKA_BROKER_ID: 2 - # KAFKA_ADVERTISED_PORT: 9094 - # KAFKA_PORT: 9092 - # KAFKA_HOST_NAME: kafka2 - # healthcheck: - # test: /kafka_check_health - # interval: 10s - # timeout: 10s - # retries: 30 - # start_period: 10s - # depends_on: - # zookeeper: - # condition: service_healthy - - # kafka3: - # image: wurstmeister/kafka:0.11.0.1 - # ports: - # - "9095:9095" - # volumes: - # - ./ssl:/ssl - # - ./scripts/kafka_check_health:/kafka_check_health - # env_file: docker-compose-kafka.env - # environment: - # KAFKA_BROKER_ID: 3 - # KAFKA_ADVERTISED_PORT: 9095 - # KAFKA_PORT: 9092 - # KAFKA_HOST_NAME: kafka3 - # healthcheck: - # test: /kafka_check_health - # interval: 10s - # timeout: 10s - # retries: 30 - # start_period: 10s - # depends_on: - # zookeeper: - # condition: service_healthy - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - name: Install Dependencies - run: mix deps.get - - name: Build Dependencies - run: mix deps.compile - - name: Compile - run: mix compile --warnings-as-errors - - name: Docker ps - run: docker ps - - name: Run Tests - run: ./scripts/ci_tests.sh diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml deleted file mode 100644 index dc9a4caf..00000000 --- a/.github/workflows/elixir.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Elixir CI - -on: [push, pull_request] - # push: - # branches: [ master ] - # pull_request: - # branches: [ master ] - - -jobs: - build: - - runs-on: ubuntu-18.04 - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - services: - zookeeper: - image: wurstmeister/zookeeper:3.4.6 - ports: - - "2181:2181" - healthcheck: - test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] - interval: 5s - timeout: 10s - retries: 15 - start_period: 10s - - kafka1: - hostname: kafka1 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9093:9093" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_PORT: 9093 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka1 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy - - kafka2: - hostname: kafka2 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9094:9094" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_PORT: 9094 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka2 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy - - kafka3: - hostname: kafka3 - image: wurstmeister/kafka:0.11.0.1 - ports: - - "9095:9095" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health - env_file: docker-compose-kafka.env - environment: - KAFKA_BROKER_ID: 3 - KAFKA_ADVERTISED_PORT: 9095 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka3 - healthcheck: - test: /kafka_check_health - interval: 10s - timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy - - steps: - - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - name: Install Dependencies - run: mix deps.get - - name: Build Dependencies - run: mix deps.compile - - name: Compile - run: mix compile --warnings-as-errors - # - name: Docker ps - # run: docker ps - - name: Run Tests - run: ./scripts/ci_tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..5eef212a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Elixir CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + + +jobs: + test-dockerized: + name: runner / Test Suite + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + - name: Install Dependencies + run: mix deps.get + - name: Build Dependencies + run: mix deps.compile + - name: Docker ps + run: docker ps + - name: Run Tests + run: ./scripts/ci_tests.sh \ No newline at end of file From 039c151a0636a5d1f13a7e90c77b50b1e38fc08c Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 10:52:37 -0600 Subject: [PATCH 12/30] Add common Checks to Github actions CI --- .github/workflows/checks.yml | 220 +++++++++++++++++++++++++++++++++++ mix.exs | 8 ++ 2 files changed, 228 insertions(+) create mode 100644 .github/workflows/checks.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000..1a608e12 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,220 @@ +name: Elixir Checks + +on: [push, pull_request] + # push: + # branches: [ master ] + # pull_request: + # branches: [ master ] + + +jobs: + setup: + name: runner / cache dependencies + runs-on: ubuntu-latest + env: + MIX_ENV: dev + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Install Mix Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mix local.rebar --force + mix local.hex --force + mix deps.get + + - name: Compile Mix Dependencies + if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' + run: mix deps.compile + + + + compile: + name: runner / Compiler warnings as errors + runs-on: ubuntu-latest + needs: [setup] + env: + MIX_ENV: dev + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Compile + run: mix compile --warnings-as-errors --force + + credo: + name: check / credo + runs-on: ubuntu-latest + needs: [setup] + env: + MIX_ENV: dev + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Credo + run: mix credo --strict + + formatter: + name: check / formatter + runs-on: ubuntu-latest + needs: [setup] + env: + MIX_ENV: dev + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Check Formatting + run: mix format --check-formatted + + dialyzer: + name: check / dialyzer + needs: [setup] + runs-on: ubuntu-latest + env: + MIX_ENV: dev + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve PLT Cache + uses: actions/cache@v1 + id: plt-cache + with: + path: priv/plts + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-plts-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Create PLTs + if: steps.plt-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix dialyzer --plt + + - name: Run dialyzer + run: mix dialyzer --no-check --halt-exit-status + + + + + \ No newline at end of file diff --git a/mix.exs b/mix.exs index 351930d0..4547cab3 100644 --- a/mix.exs +++ b/mix.exs @@ -18,6 +18,7 @@ defmodule KafkaEx.Mixfile do description: description(), package: package(), deps: deps(), + dialyzer: dialyzer(), docs: [ main: "readme", extras: [ @@ -68,4 +69,11 @@ defmodule KafkaEx.Mixfile do links: %{"Github" => "https://github.com/kafkaex/kafka_ex"} ] end + + defp dialyzer do + [ + plt_core_path: "priv/plts", + plt_file: {:no_warn, "priv/plts/dialyzer.plt"} + ] + end end From 51a15746b7337db718cb0472f6e839cb91b46d20 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:23:34 -0600 Subject: [PATCH 13/30] Split integration test stages up --- .github/workflows/test.yml | 384 ++++++++++++++++++++++++++++++++++--- 1 file changed, 362 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5eef212a..0d257cae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,303 @@ name: Elixir CI -on: +on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] - + branches: [master] jobs: - test-dockerized: - name: runner / Test Suite + setup: + name: runner / setup + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Install Mix Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mix local.rebar --force + mix local.hex --force + mix deps.get + + - name: Compile Mix Dependencies + if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' + run: mix deps.compile + + test-integration: + name: runner / Test Integration + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include integration + + test-consumer_group: + name: runner / Test consumer_group + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include consumer_group + test-server_0_p_10_and_later: + name: runner / Test server_0_p_10_and_later + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include server_0_p_10_and_later + test-server_0_p_10_p_1: + name: runner / Test server_0_p_10_p_1 + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include server_0_p_10_p_1 + test-server_0_p_9_p_0: + name: runner / Test server_0_p_9_p_0 + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include server_0_p_9_p_0 + test-server_0_p_8_p_0: + name: runner / Test server_0_p_8_p_0 + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include server_0_p_8_p_0 + test-new_client: + name: runner / Test new_client runs-on: ubuntu-latest env: @@ -22,19 +310,71 @@ jobs: erlang: 22.3 steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - name: Install Dependencies - run: mix deps.get - - name: Build Dependencies - run: mix deps.compile - - name: Docker ps - run: docker ps - - name: Run Tests - run: ./scripts/ci_tests.sh \ No newline at end of file + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test --include new_client + test: + name: runner / Test new_client + + runs-on: ubuntu-latest + env: + MIX_ENV: test + + strategy: + matrix: + include: + - elixir: 1.10.2 + erlang: 22.3 + + steps: + - uses: actions/checkout@v2 + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Setup elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker ps + run: docker ps + - name: Run Tests + run: mix test From 8281368cfc66d3d6108e52849ffa8606c0d7b2ec Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:24:55 -0600 Subject: [PATCH 14/30] run test suite on push for testing --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d257cae..aafa71a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,10 @@ -name: Elixir CI +name: Test Runner -on: - push: - branches: [master] - pull_request: - branches: [master] +on: [push, pull_request] + # push: + # branches: [master] + # pull_request: + # branches: [master] jobs: setup: From 94dd522fca34dee173b7d481c25d0f8fec4811f4 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:26:35 -0600 Subject: [PATCH 15/30] test - wait for setup stage to cache deps --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aafa71a6..e68a653a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,7 @@ jobs: test-integration: name: runner / Test Integration + needs: [setup] runs-on: ubuntu-latest env: @@ -97,6 +98,7 @@ jobs: runs-on: ubuntu-latest env: MIX_ENV: test + needs: [setup] strategy: matrix: @@ -134,6 +136,7 @@ jobs: run: mix test --include consumer_group test-server_0_p_10_and_later: name: runner / Test server_0_p_10_and_later + needs: [setup] runs-on: ubuntu-latest env: @@ -175,6 +178,7 @@ jobs: run: mix test --include server_0_p_10_and_later test-server_0_p_10_p_1: name: runner / Test server_0_p_10_p_1 + needs: [setup] runs-on: ubuntu-latest env: @@ -216,6 +220,7 @@ jobs: run: mix test --include server_0_p_10_p_1 test-server_0_p_9_p_0: name: runner / Test server_0_p_9_p_0 + needs: [setup] runs-on: ubuntu-latest env: @@ -257,6 +262,7 @@ jobs: run: mix test --include server_0_p_9_p_0 test-server_0_p_8_p_0: name: runner / Test server_0_p_8_p_0 + needs: [setup] runs-on: ubuntu-latest env: @@ -298,6 +304,7 @@ jobs: run: mix test --include server_0_p_8_p_0 test-new_client: name: runner / Test new_client + needs: [setup] runs-on: ubuntu-latest env: @@ -339,6 +346,7 @@ jobs: run: mix test --include new_client test: name: runner / Test new_client + needs: [setup] runs-on: ubuntu-latest env: From 888feefbeead81ab1d172413dfc042d325fdd6e0 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:31:29 -0600 Subject: [PATCH 16/30] Try running test suite args with matrix --- .github/workflows/test.yml | 604 +++++++++++++++++++------------------ 1 file changed, 307 insertions(+), 297 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e68a653a..2c7f4ec0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,303 +49,8 @@ jobs: if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' run: mix deps.compile - test-integration: - name: runner / Test Integration - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include integration - - test-consumer_group: - name: runner / Test consumer_group - - runs-on: ubuntu-latest - env: - MIX_ENV: test - needs: [setup] - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include consumer_group - test-server_0_p_10_and_later: - name: runner / Test server_0_p_10_and_later - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include server_0_p_10_and_later - test-server_0_p_10_p_1: - name: runner / Test server_0_p_10_p_1 - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include server_0_p_10_p_1 - test-server_0_p_9_p_0: - name: runner / Test server_0_p_9_p_0 - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include server_0_p_9_p_0 - test-server_0_p_8_p_0: - name: runner / Test server_0_p_8_p_0 - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include server_0_p_8_p_0 - test-new_client: - name: runner / Test new_client - needs: [setup] - - runs-on: ubuntu-latest - env: - MIX_ENV: test - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Docker ps - run: docker ps - - name: Run Tests - run: mix test --include new_client test: - name: runner / Test new_client + name: runner / Test Integration needs: [setup] runs-on: ubuntu-latest @@ -354,6 +59,16 @@ jobs: strategy: matrix: + test_args: [ + "--include integration", + "--include consumer_group", + "--include server_0_p_10_and_later", + "--include server_0_p_10_p_1", + "--include server_0_p_9_p_0", + "--include server_0_p_8_p_0", + "--include new_client", + "" + ] include: - elixir: 1.10.2 erlang: 22.3 @@ -385,4 +100,299 @@ jobs: - name: Docker ps run: docker ps - name: Run Tests - run: mix test + run: mix test ${{ matrix.test_args }} + + # test-consumer_group: + # name: runner / Test consumer_group + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + # needs: [setup] + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include consumer_group + # test-server_0_p_10_and_later: + # name: runner / Test server_0_p_10_and_later + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include server_0_p_10_and_later + # test-server_0_p_10_p_1: + # name: runner / Test server_0_p_10_p_1 + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include server_0_p_10_p_1 + # test-server_0_p_9_p_0: + # name: runner / Test server_0_p_9_p_0 + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include server_0_p_9_p_0 + # test-server_0_p_8_p_0: + # name: runner / Test server_0_p_8_p_0 + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include server_0_p_8_p_0 + # test-new_client: + # name: runner / Test new_client + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test --include new_client + # test: + # name: runner / Test new_client + # needs: [setup] + + # runs-on: ubuntu-latest + # env: + # MIX_ENV: test + + # strategy: + # matrix: + # include: + # - elixir: 1.10.2 + # erlang: 22.3 + + # steps: + # - uses: actions/checkout@v2 + # - name: Docker-compose up + # run: ./scripts/docker_up.sh + # - name: Setup elixir + # uses: actions/setup-elixir@v1 + # with: + # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + # otp-version: ${{matrix.erlang}} # Define the OTP version [required] + + # - name: Retrieve Mix Dependencies Cache + # uses: actions/cache@v2 + # id: mix-cache # id to use in retrieve action + # with: + # path: deps + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Retrieve Mix Dependencies Compilation Cache + # uses: actions/cache@v2 + # id: mix-deps-compile-cache # id to use in retrieve action + # with: + # path: _build + # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + # - name: Docker ps + # run: docker ps + # - name: Run Tests + # run: mix test From 729df7c8c0a3d01c995a4cb3fe9d7aa0ad609b52 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:37:26 -0600 Subject: [PATCH 17/30] run Suit checks via matrix too --- .github/workflows/checks.yml | 85 +++--------------------------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 067d2fe6..d61e2a4c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -54,7 +54,7 @@ jobs: compile: - name: check / compile + name: check runs-on: ubuntu-latest needs: [setup] env: @@ -65,6 +65,10 @@ jobs: include: - elixir: 1.10.2 erlang: 22.3 + checks: + - credo --struct + - compile --warnings-as-errors --force + - format --check-formatted steps: - uses: actions/checkout@v2 @@ -87,83 +91,8 @@ jobs: path: _build key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - name: Compile - run: mix compile --warnings-as-errors --force - - credo: - name: check / credo - runs-on: ubuntu-latest - needs: [setup] - env: - MIX_ENV: dev - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Credo - run: mix credo --strict - - formatter: - name: check / formatter - runs-on: ubuntu-latest - needs: [setup] - env: - MIX_ENV: dev - - strategy: - matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Check Formatting - run: mix format --check-formatted - + - run: mix ${{ matrix.checks }} + dialyzer: name: check / dialyzer needs: [setup] From 0526100747bf09e2b9ae0cc96f972ab2b20e554f Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:39:30 -0600 Subject: [PATCH 18/30] Disable fast failures on test matrix --- .github/workflows/checks.yml | 1 + .github/workflows/test.yml | 67 +++++++----------------------------- 2 files changed, 14 insertions(+), 54 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d61e2a4c..8a260a98 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,6 +61,7 @@ jobs: MIX_ENV: dev strategy: + fail-fast: false matrix: include: - elixir: 1.10.2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c7f4ec0..4c6b1a33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,7 @@ name: Test Runner -on: [push, pull_request] +on: + [push, pull_request] # push: # branches: [master] # pull_request: @@ -58,17 +59,18 @@ jobs: MIX_ENV: test strategy: + fail-fast: false matrix: - test_args: [ - "--include integration", - "--include consumer_group", - "--include server_0_p_10_and_later", - "--include server_0_p_10_p_1", - "--include server_0_p_9_p_0", - "--include server_0_p_8_p_0", - "--include new_client", - "" - ] + test_args: + - "--include integration" + - "--include consumer_group" + - "--include server_0_p_10_and_later" + - "--include server_0_p_10_p_1" + - "--include server_0_p_9_p_0" + - "--include server_0_p_8_p_0" + - "--include new_client" + - "" + include: - elixir: 1.10.2 erlang: 22.3 @@ -101,21 +103,17 @@ jobs: run: docker ps - name: Run Tests run: mix test ${{ matrix.test_args }} - # test-consumer_group: # name: runner / Test consumer_group - # runs-on: ubuntu-latest # env: # MIX_ENV: test # needs: [setup] - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -125,21 +123,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -147,17 +142,14 @@ jobs: # test-server_0_p_10_and_later: # name: runner / Test server_0_p_10_and_later # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -167,21 +159,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -189,17 +178,14 @@ jobs: # test-server_0_p_10_p_1: # name: runner / Test server_0_p_10_p_1 # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -209,21 +195,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -231,17 +214,14 @@ jobs: # test-server_0_p_9_p_0: # name: runner / Test server_0_p_9_p_0 # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -251,21 +231,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -273,17 +250,14 @@ jobs: # test-server_0_p_8_p_0: # name: runner / Test server_0_p_8_p_0 # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -293,21 +267,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -315,17 +286,14 @@ jobs: # test-new_client: # name: runner / Test new_client # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -335,21 +303,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests @@ -357,17 +322,14 @@ jobs: # test: # name: runner / Test new_client # needs: [setup] - # runs-on: ubuntu-latest # env: # MIX_ENV: test - # strategy: # matrix: # include: # - elixir: 1.10.2 # erlang: 22.3 - # steps: # - uses: actions/checkout@v2 # - name: Docker-compose up @@ -377,21 +339,18 @@ jobs: # with: # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache # uses: actions/cache@v2 # id: mix-cache # id to use in retrieve action # with: # path: deps # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache # uses: actions/cache@v2 # id: mix-deps-compile-cache # id to use in retrieve action # with: # path: _build # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps # run: docker ps # - name: Run Tests From 01e9ee9b31a299b7ac71340ff35ca47e12910a82 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:40:50 -0600 Subject: [PATCH 19/30] Correct credo runner command --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8a260a98..df3ec749 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -54,7 +54,7 @@ jobs: compile: - name: check + name: checks runs-on: ubuntu-latest needs: [setup] env: @@ -67,7 +67,7 @@ jobs: - elixir: 1.10.2 erlang: 22.3 checks: - - credo --struct + - credo - compile --warnings-as-errors --force - format --check-formatted From 51cb92dad81ad4ec7ae4c818120de15bad7f38d8 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 11:50:36 -0600 Subject: [PATCH 20/30] improve naming and specify complete command in matrix --- .github/workflows/checks.yml | 44 +++--- .github/workflows/test.yml | 285 ++--------------------------------- 2 files changed, 34 insertions(+), 295 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index df3ec749..15ffbf2f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,11 +1,10 @@ -name: Elixir Checks - -on: [push, pull_request] - # push: - # branches: [ master ] - # pull_request: - # branches: [ master ] +name: CI Checks +on: + push: + branches: [master] + pull_request: + branches: [master] jobs: setup: @@ -33,7 +32,7 @@ jobs: with: path: deps key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - + - name: Retrieve Mix Dependencies Compilation Cache uses: actions/cache@v2 id: mix-deps-compile-cache # id to use in retrieve action @@ -51,10 +50,8 @@ jobs: if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' run: mix deps.compile - - - compile: - name: checks + run-checks: + name: check runs-on: ubuntu-latest needs: [setup] env: @@ -67,9 +64,9 @@ jobs: - elixir: 1.10.2 erlang: 22.3 checks: - - credo - - compile --warnings-as-errors --force - - format --check-formatted + - mix credo + - mix compile --warnings-as-errors --force + - mix format --check-formatted steps: - uses: actions/checkout@v2 @@ -84,16 +81,16 @@ jobs: with: path: deps key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - + - name: Retrieve Mix Dependencies Compilation Cache uses: actions/cache@v2 id: mix-deps-compile-cache # id to use in retrieve action with: path: _build key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - run: mix ${{ matrix.checks }} - + + - run: ${{ matrix.checks }} + dialyzer: name: check / dialyzer needs: [setup] @@ -120,14 +117,14 @@ jobs: with: path: deps key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - + - name: Retrieve Mix Dependencies Compilation Cache uses: actions/cache@v2 id: mix-deps-compile-cache # id to use in retrieve action with: path: _build key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - + - name: Retrieve PLT Cache uses: actions/cache@v1 id: plt-cache @@ -143,8 +140,3 @@ jobs: - name: Run dialyzer run: mix dialyzer --no-check --halt-exit-status - - - - - \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c6b1a33..850bdbbb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,10 @@ -name: Test Runner +name: CI Tests on: - [push, pull_request] - # push: - # branches: [master] - # pull_request: - # branches: [master] + push: + branches: [master] + pull_request: + branches: [master] jobs: setup: @@ -51,7 +50,7 @@ jobs: run: mix deps.compile test: - name: runner / Test Integration + name: runner / Test needs: [setup] runs-on: ubuntu-latest @@ -61,15 +60,15 @@ jobs: strategy: fail-fast: false matrix: - test_args: - - "--include integration" - - "--include consumer_group" - - "--include server_0_p_10_and_later" - - "--include server_0_p_10_p_1" - - "--include server_0_p_9_p_0" - - "--include server_0_p_8_p_0" - - "--include new_client" - - "" + tests: + - mix test --include integration + - mix test --include consumer_group + - mix test --include server_0_p_10_and_later + - mix test --include server_0_p_10_p_1 + - mix test --include server_0_p_9_p_0 + - mix test --include server_0_p_8_p_0 + - mix test --include new_client + - mix test include: - elixir: 1.10.2 @@ -102,256 +101,4 @@ jobs: - name: Docker ps run: docker ps - name: Run Tests - run: mix test ${{ matrix.test_args }} - # test-consumer_group: - # name: runner / Test consumer_group - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # needs: [setup] - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include consumer_group - # test-server_0_p_10_and_later: - # name: runner / Test server_0_p_10_and_later - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include server_0_p_10_and_later - # test-server_0_p_10_p_1: - # name: runner / Test server_0_p_10_p_1 - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include server_0_p_10_p_1 - # test-server_0_p_9_p_0: - # name: runner / Test server_0_p_9_p_0 - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include server_0_p_9_p_0 - # test-server_0_p_8_p_0: - # name: runner / Test server_0_p_8_p_0 - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include server_0_p_8_p_0 - # test-new_client: - # name: runner / Test new_client - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test --include new_client - # test: - # name: runner / Test new_client - # needs: [setup] - # runs-on: ubuntu-latest - # env: - # MIX_ENV: test - # strategy: - # matrix: - # include: - # - elixir: 1.10.2 - # erlang: 22.3 - # steps: - # - uses: actions/checkout@v2 - # - name: Docker-compose up - # run: ./scripts/docker_up.sh - # - name: Setup elixir - # uses: actions/setup-elixir@v1 - # with: - # elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - # otp-version: ${{matrix.erlang}} # Define the OTP version [required] - # - name: Retrieve Mix Dependencies Cache - # uses: actions/cache@v2 - # id: mix-cache # id to use in retrieve action - # with: - # path: deps - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Retrieve Mix Dependencies Compilation Cache - # uses: actions/cache@v2 - # id: mix-deps-compile-cache # id to use in retrieve action - # with: - # path: _build - # key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - # - name: Docker ps - # run: docker ps - # - name: Run Tests - # run: mix test + run: ${{ matrix.tests }} From 4ee2cfc8f7e5fd8d3a617deb71eb9375dd065909 Mon Sep 17 00:00:00 2001 From: Grant McLendon Date: Thu, 7 Jan 2021 13:08:30 -0600 Subject: [PATCH 21/30] try tweaking kafka config to use fewer threads --- docker-compose-kafka.env | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker-compose-kafka.env b/docker-compose-kafka.env index f1bbcf91..fd7f74b9 100644 --- a/docker-compose-kafka.env +++ b/docker-compose-kafka.env @@ -1,3 +1,4 @@ + ################################# # Common Kafka config ################################# @@ -33,14 +34,16 @@ KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG ######## Config KAFKA_DELETE_TOPIC_ENABLE=true -# KAFKA_NUM_NETWORK_THREADS=3 -# KAFKA_NUM_IO_THREADS=8 +KAFKA_NUM_NETWORK_THREADS=1 +KAFKA_NUM_IO_THREADS=1 +KAFKA_BACKGROUND_THREADS=1 +KAFKA_LOG_CLEANER_THREADS=0 # KAFKA_SOCKET_SEND_BUFFER_BYTES=102400 # KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400 # KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 KAFKA_LOG_DIRS=/tmp/kafka_log KAFKA_NUM_PARTITIONS=1 KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1 -KAFKA_LOG_RETENTION_HOURS=168 +KAFKA_LOG_RETENTION_HOURS=1 # KAFKA_LOG_SEGMENT_BYTES=1073741824 # KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000 From e2822dbdf559b2351c026ea54bb986bb672ee063 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 14 Apr 2022 10:31:06 +0200 Subject: [PATCH 22/30] Update workflows to work with OTP - Lock ubuntu version to 20.04 - Steps names to make it more explicit - Fix OTP --- .github/workflows/checks.yml | 61 ++++++++++++++++++------------------ .github/workflows/test.yml | 41 ++++++++++++++---------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 15ffbf2f..195c6731 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -2,57 +2,54 @@ name: CI Checks on: push: - branches: [master] + branches: [ master ] pull_request: - branches: [master] + branches: [ master ] jobs: - setup: - name: check / setup - runs-on: ubuntu-latest + dependencies: + name: check | setup dependencies + runs-on: ubuntu-20.04 env: MIX_ENV: dev ACTIONS_ALLOW_UNSECURE_COMMANDS: true - strategy: matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 + elixir: ['1.12.3'] + erlang: ['24.3'] + steps: - - uses: actions/checkout@v2 - - uses: actions/setup-elixir@v1 + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Github repo + uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] - - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + - name: Retrieve cached dependencies uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action + id: mix-cache with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - name: Install Mix Dependencies + path: | + deps + _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} + - name: Install dependencies if: steps.mix-cache.outputs.cache-hit != 'true' run: | mix local.rebar --force mix local.hex --force mix deps.get - - - name: Compile Mix Dependencies - if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' - run: mix deps.compile + mix deps.compile run-checks: - name: check - runs-on: ubuntu-latest + name: check + runs-on: ubuntu-20.04 needs: [setup] env: MIX_ENV: dev @@ -70,10 +67,12 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-elixir@v1 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Mix Dependencies Cache uses: actions/cache@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 850bdbbb..cb86d102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,28 +2,34 @@ name: CI Tests on: push: - branches: [master] + branches: [ master ] pull_request: - branches: [master] + branches: [ master ] jobs: setup: - name: runner / setup - runs-on: ubuntu-latest + name: test / setup + runs-on: ubuntu-20.04 env: MIX_ENV: test - strategy: matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 + elixir: ['1.12.3'] + erlang: ['24.3'] + steps: - - uses: actions/checkout@v2 - - uses: actions/setup-elixir@v1 + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Github repo + uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Mix Dependencies Cache uses: actions/cache@v2 @@ -53,7 +59,7 @@ jobs: name: runner / Test needs: [setup] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: MIX_ENV: test @@ -71,18 +77,17 @@ jobs: - mix test include: - - elixir: 1.10.2 - erlang: 22.3 + - elixir: 1.12.3 + erlang: 24.3 steps: - uses: actions/checkout@v2 - - name: Docker-compose up - run: ./scripts/docker_up.sh - - name: Setup elixir + - name: Setup elixir & erlang enviroment uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Mix Dependencies Cache uses: actions/cache@v2 @@ -98,7 +103,9 @@ jobs: path: _build key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Docker-compose up + run: ./scripts/docker_up.sh - name: Docker ps - run: docker ps + run: docker ps -a - name: Run Tests run: ${{ matrix.tests }} From 9ad7bd4a102b8e0133a674f5ae1e9a4497c1a634 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 21 Apr 2022 13:06:31 +0200 Subject: [PATCH 23/30] Update to confluent kafka --- .github/workflows/test.yml | 12 +- config/config.exs | 6 +- docker-compose-kafka.env | 19 +-- docker-compose.yml | 115 ++++++++---------- scripts/kafka_check_health | 21 ---- ...keystore.jks => kafka.server.keystore.jks} | Bin ...tstore.jks => kafka.server.truststore.jks} | Bin 7 files changed, 60 insertions(+), 113 deletions(-) delete mode 100755 scripts/kafka_check_health rename ssl/{server.keystore.jks => kafka.server.keystore.jks} (100%) rename ssl/{server.truststore.jks => kafka.server.truststore.jks} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb86d102..3d215b26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,16 +66,6 @@ jobs: strategy: fail-fast: false matrix: - tests: - - mix test --include integration - - mix test --include consumer_group - - mix test --include server_0_p_10_and_later - - mix test --include server_0_p_10_p_1 - - mix test --include server_0_p_9_p_0 - - mix test --include server_0_p_8_p_0 - - mix test --include new_client - - mix test - include: - elixir: 1.12.3 erlang: 24.3 @@ -108,4 +98,4 @@ jobs: - name: Docker ps run: docker ps -a - name: Run Tests - run: ${{ matrix.tests }} + run: ./scripts/all_tests.sh diff --git a/config/config.exs b/config/config.exs index 20d53e81..7d44b30a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,9 +14,9 @@ config :kafka_ex, # server.properties file. # In the case below you would set "advertised.host.name=localhost" brokers: [ + {"localhost", 9092}, {"localhost", 9093}, - {"localhost", 9094}, - {"localhost", 9095} + {"localhost", 9094} ], # # OR: @@ -61,6 +61,8 @@ config :kafka_ex, # see SSL OPTION DESCRIPTIONS - CLIENT SIDE at http://erlang.org/doc/man/ssl.html # for supported options ssl_options: [ + # Fix warnings. More at https://github.com/erlang/otp/issues/5352 + verify: :verify_none, cacertfile: File.cwd!() <> "/ssl/ca-cert", certfile: File.cwd!() <> "/ssl/cert.pem", keyfile: File.cwd!() <> "/ssl/key.pem" diff --git a/docker-compose-kafka.env b/docker-compose-kafka.env index fd7f74b9..28879e3c 100644 --- a/docker-compose-kafka.env +++ b/docker-compose-kafka.env @@ -1,14 +1,9 @@ - ################################# # Common Kafka config ################################# # Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`. -######## topic creation - -KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 - ######## zookeeper KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 @@ -17,19 +12,17 @@ KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000 ######## advertised hosts names and protocols # alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*" -KAFKA_ADVERTISED_HOST_NAME=localhost -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:SSL -KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE -KAFKA_PROTOCOL_NAME=INSIDE +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:SSL +KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL ######## SSL -KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks +KAFKA_SSL_KEYSTORE_LOCATION=/etc/kafka/secrets/kafka.server.keystore.jks KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex KAFKA_SSL_KEY_PASSWORD=kafka_ex -KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks + +KAFKA_SSL_TRUSTSTORE_LOCATION=/etc/kafka/secrets/kafka.server.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex -KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG ######## Config @@ -37,7 +30,7 @@ KAFKA_DELETE_TOPIC_ENABLE=true KAFKA_NUM_NETWORK_THREADS=1 KAFKA_NUM_IO_THREADS=1 KAFKA_BACKGROUND_THREADS=1 -KAFKA_LOG_CLEANER_THREADS=0 +KAFKA_LOG_CLEANER_THREADS=1 # KAFKA_SOCKET_SEND_BUFFER_BYTES=102400 # KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400 # KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 diff --git a/docker-compose.yml b/docker-compose.yml index 58fb76ec..ef1682f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,98 +1,81 @@ -version: '2.4' +version: "3.8" services: zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: confluentinc/cp-zookeeper:5.5.1 ports: - - "2181:2181" - healthcheck: - test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] - interval: 5s - timeout: 10s - retries: 15 - start_period: 10s + - '32181:32181' + environment: + ZOOKEEPER_CLIENT_PORT: 32181 + ZOOKEEPER_TICK_TIME: 2000 - kafka1: - hostname: kafka1 - image: wurstmeister/kafka:0.11.0.1 + kafka-1: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9093:9093" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health + - '9092:9092' + depends_on: + - zookeeper env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_PORT: 9093 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-1:29092 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - kafka2: - hostname: kafka2 - image: wurstmeister/kafka:0.11.0.1 + kafka-2: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9094:9094" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health + - '9093:9093' + depends_on: + - zookeeper env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_PORT: 9094 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29093,EXTERNAL://localhost:9093 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-2:29093 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - kafka3: - hostname: kafka3 - image: wurstmeister/kafka:0.11.0.1 + kafka-3: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9095:9095" - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health + - '9094:9094' + depends_on: + - zookeeper env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 3 - KAFKA_ADVERTISED_PORT: 9095 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29094,EXTERNAL://localhost:9094 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-2:29093 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - # This is a dummy service that forces all other services to be healthy before - # docker-compose up can be considered successful. ready: image: busybox:1.31-musl command: tail -f /dev/null depends_on: - kafka1: - condition: service_healthy - kafka2: + kafka-1: condition: service_healthy - kafka3: + kafka-2: condition: service_healthy + kafka-3: + condition: service_healthy \ No newline at end of file diff --git a/scripts/kafka_check_health b/scripts/kafka_check_health deleted file mode 100755 index b8b7eba7..00000000 --- a/scripts/kafka_check_health +++ /dev/null @@ -1,21 +0,0 @@ -#! /bin/bash - -cd $(dirname $0) - -broker_ready() { - # Inspired by: https://github.com/wurstmeister/kafka-docker/issues/167#issuecomment-439849789 - current_broker=$(cat $KAFKA_HOME/config/server.properties | awk 'BEGIN{FS="="}/^broker.id=/{print $2}') - brokers=$($KAFKA_HOME/bin/zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" | tail -1) - - echo "brokers: $brokers" - echo "current_broker: $current_broker" - - echo "${brokers}" | jq '.[]' | grep -q "^${current_broker}$"; - return $? -} - -####################### - -broker_ready - -####################### diff --git a/ssl/server.keystore.jks b/ssl/kafka.server.keystore.jks similarity index 100% rename from ssl/server.keystore.jks rename to ssl/kafka.server.keystore.jks diff --git a/ssl/server.truststore.jks b/ssl/kafka.server.truststore.jks similarity index 100% rename from ssl/server.truststore.jks rename to ssl/kafka.server.truststore.jks From 52568e7180d4e006030f3c1bd955ed22fdf0f0ee Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 21 Apr 2022 15:27:13 +0200 Subject: [PATCH 24/30] MInor fixes - Update ssl related code and nest it into describes - Updates tests to support new broker version - Update docker-compose to create topics - fix check --- .github/workflows/checks.yml | 86 +++++---- docker-compose.yml | 10 +- lib/kafka_ex/gen_consumer.ex | 4 +- lib/kafka_ex/network_client.ex | 16 +- lib/kafka_ex/protocol/fetch.ex | 4 +- lib/kafka_ex/server.ex | 36 +--- lib/kafka_ex/server_0_p_8_p_2.ex | 4 +- mix.exs | 2 +- mix.lock | 4 +- .../compatibility_0_p_10_and_later_test.exs | 80 ++++---- test/integration/server0_p_10_p_1_test.exs | 80 ++++---- test/socket_test.exs | 172 ++++++++++-------- 12 files changed, 257 insertions(+), 241 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 195c6731..d909801c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,51 +19,55 @@ jobs: erlang: ['24.3'] steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - name: Checkout Github repo - uses: actions/checkout@v2 - - name: Setup elixir & erlang enviroment - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - - name: Retrieve cached dependencies - uses: actions/cache@v2 - id: mix-cache - with: - path: | - deps - _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} - - name: Install dependencies - if: steps.mix-cache.outputs.cache-hit != 'true' - run: | - mix local.rebar --force - mix local.hex --force - mix deps.get - mix deps.compile + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Github repo + uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.erlang}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Install Mix Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mix local.rebar --force + mix local.hex --force + mix deps.get + + - name: Compile Mix Dependencies + if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' + run: mix deps.compile run-checks: name: check runs-on: ubuntu-20.04 - needs: [setup] + needs: [dependencies] env: MIX_ENV: dev strategy: fail-fast: false matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 - checks: - - mix credo - - mix compile --warnings-as-errors --force - - mix format --check-formatted + elixir: ['1.12.3'] + erlang: ['24.3'] steps: - uses: actions/checkout@v2 @@ -88,20 +92,22 @@ jobs: path: _build key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - run: ${{ matrix.checks }} + - run: | + mix compile --force + mix credo + mix format --check-formatted dialyzer: name: check / dialyzer - needs: [setup] + needs: [dependencies] runs-on: ubuntu-latest env: MIX_ENV: dev strategy: matrix: - include: - - elixir: 1.10.2 - erlang: 22.3 + elixir: ['1.12.3'] + erlang: ['24.3'] steps: - uses: actions/checkout@v2 diff --git a/docker-compose.yml b/docker-compose.yml index ef1682f5..42329210 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,13 +69,15 @@ services: - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - ready: - image: busybox:1.31-musl - command: tail -f /dev/null + kafka_setup: + image: confluentinc/cp-kafka:5.5.1 depends_on: kafka-1: condition: service_healthy kafka-2: condition: service_healthy kafka-3: - condition: service_healthy \ No newline at end of file + condition: service_healthy + command: "bash -c 'kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic consumer_group_implementation_test && \ + kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic test0p8p0 && \ + kafka-topics --zookeeper zookeeper:32181 --list'" diff --git a/lib/kafka_ex/gen_consumer.ex b/lib/kafka_ex/gen_consumer.ex index 180affd0..41380ad9 100644 --- a/lib/kafka_ex/gen_consumer.ex +++ b/lib/kafka_ex/gen_consumer.ex @@ -810,9 +810,7 @@ defmodule KafkaEx.GenConsumer do KafkaEx.latest_offset(topic, partition, worker_name) _ -> - raise "Offset out of range while consuming topic #{topic}, partition #{ - partition - }." + raise "Offset out of range while consuming topic #{topic}, partition #{partition}." end %State{ diff --git a/lib/kafka_ex/network_client.ex b/lib/kafka_ex/network_client.ex index 296c84d8..f38f3cc6 100644 --- a/lib/kafka_ex/network_client.ex +++ b/lib/kafka_ex/network_client.ex @@ -25,9 +25,7 @@ defmodule KafkaEx.NetworkClient do err -> Logger.log( :error, - "Could not connect to broker #{inspect(host)}:#{inspect(port)} because of error #{ - inspect(err) - }" + "Could not connect to broker #{inspect(host)}:#{inspect(port)} because of error #{inspect(err)}" ) nil @@ -50,9 +48,7 @@ defmodule KafkaEx.NetworkClient do {_, reason} -> Logger.log( :error, - "Asynchronously sending data to broker #{inspect(broker.host)}:#{ - inspect(broker.port) - } failed with #{inspect(reason)}" + "Asynchronously sending data to broker #{inspect(broker.host)}:#{inspect(broker.port)} failed with #{inspect(reason)}" ) reason @@ -77,9 +73,7 @@ defmodule KafkaEx.NetworkClient do {:error, reason} -> Logger.log( :error, - "Receiving data from broker #{inspect(broker.host)}:#{ - inspect(broker.port) - } failed with #{inspect(reason)}" + "Receiving data from broker #{inspect(broker.host)}:#{inspect(broker.port)} failed with #{inspect(reason)}" ) Socket.close(socket) @@ -90,9 +84,7 @@ defmodule KafkaEx.NetworkClient do {_, reason} -> Logger.log( :error, - "Sending data to broker #{inspect(broker.host)}:#{ - inspect(broker.port) - } failed with #{inspect(reason)}" + "Sending data to broker #{inspect(broker.host)}:#{inspect(broker.port)} failed with #{inspect(reason)}" ) Socket.close(socket) diff --git a/lib/kafka_ex/protocol/fetch.ex b/lib/kafka_ex/protocol/fetch.ex index 8a704746..6f8aea3d 100644 --- a/lib/kafka_ex/protocol/fetch.ex +++ b/lib/kafka_ex/protocol/fetch.ex @@ -162,9 +162,7 @@ defmodule KafkaEx.Protocol.Fetch do ) when byte_size(partial_message_data) < msg_size do raise RuntimeError, - "Insufficient data fetched at offset #{offset}. Message size is #{ - msg_size - } but only received #{byte_size(partial_message_data)} bytes. Try increasing max_bytes." + "Insufficient data fetched at offset #{offset}. Message size is #{msg_size} but only received #{byte_size(partial_message_data)} bytes. Try increasing max_bytes." end # handles the single message case and the batch (compression) case diff --git a/lib/kafka_ex/server.ex b/lib/kafka_ex/server.ex index 7f1f274b..7b210408 100644 --- a/lib/kafka_ex/server.ex +++ b/lib/kafka_ex/server.ex @@ -373,9 +373,7 @@ defmodule KafkaEx.Server do def terminate(reason, state) do Logger.log( :debug, - "Shutting down worker #{inspect(state.worker_name)}, reason: #{ - inspect(reason) - }" + "Shutting down worker #{inspect(state.worker_name)}, reason: #{inspect(reason)}" ) if state.event_pid do @@ -471,9 +469,7 @@ defmodule KafkaEx.Server do nil -> Logger.log( :error, - "kafka_server_produce_send_request: leader for topic #{ - produce_request.topic - }/#{produce_request.partition} is not available" + "kafka_server_produce_send_request: leader for topic #{produce_request.topic}/#{produce_request.partition} is not available" ) :leader_not_available @@ -678,9 +674,7 @@ defmodule KafkaEx.Server do ) do Logger.log( :error, - "Metadata request for topic #{inspect(topic)} failed with error_code #{ - inspect(error_code) - }" + "Metadata request for topic #{inspect(topic)} failed with error_code #{inspect(error_code)}" ) {correlation_id, %Metadata.Response{}} @@ -731,9 +725,7 @@ defmodule KafkaEx.Server do end else message = - "Unable to fetch metadata from any brokers. Timeout is #{ - sync_timeout - }." + "Unable to fetch metadata from any brokers. Timeout is #{sync_timeout}." Logger.log(:error, message) raise message @@ -862,9 +854,7 @@ defmodule KafkaEx.Server do case broker do nil -> Logger.error(fn -> - "network_request: leader for topic #{request.topic}/#{ - request.partition - } is not available" + "network_request: leader for topic #{request.topic}/#{request.partition} is not available" end) {{:error, :topic_not_found}, updated_state} @@ -891,15 +881,11 @@ defmodule KafkaEx.Server do rescue _ -> Logger.error( - "Failed to parse a response from the server: #{ - inspect(response) - }" + "Failed to parse a response from the server: #{inspect(response)}" ) Kernel.reraise( - "Parse error during #{inspect(module)}.parse_response. Couldn't parse: #{ - inspect(response) - }", + "Parse error during #{inspect(module)}.parse_response. Couldn't parse: #{inspect(response)}", System.stacktrace() ) end @@ -932,9 +918,7 @@ defmodule KafkaEx.Server do Enum.each(brokers_to_remove, fn broker -> Logger.log( :debug, - "Closing connection to broker #{broker.node_id}: #{ - inspect(broker.host) - } on port #{inspect(broker.port)}" + "Closing connection to broker #{broker.node_id}: #{inspect(broker.host)} on port #{inspect(broker.port)}" ) NetworkClient.close_socket(broker.socket) @@ -956,9 +940,7 @@ defmodule KafkaEx.Server do nil -> Logger.log( :debug, - "Establishing connection to broker #{metadata_broker.node_id}: #{ - inspect(metadata_broker.host) - } on port #{inspect(metadata_broker.port)}" + "Establishing connection to broker #{metadata_broker.node_id}: #{inspect(metadata_broker.host)} on port #{inspect(metadata_broker.port)}" ) add_new_brokers( diff --git a/lib/kafka_ex/server_0_p_8_p_2.ex b/lib/kafka_ex/server_0_p_8_p_2.ex index 9042ee11..fed8b829 100644 --- a/lib/kafka_ex/server_0_p_8_p_2.ex +++ b/lib/kafka_ex/server_0_p_8_p_2.ex @@ -226,9 +226,7 @@ defmodule KafkaEx.Server0P8P2 do ) do Logger.log( :error, - "Fetching consumer_group #{consumer_group} metadata failed with error_code #{ - inspect(error_code) - }" + "Fetching consumer_group #{consumer_group} metadata failed with error_code #{inspect(error_code)}" ) {%ConsumerMetadataResponse{error_code: error_code}, state} diff --git a/mix.exs b/mix.exs index 4547cab3..13a8aa46 100644 --- a/mix.exs +++ b/mix.exs @@ -42,7 +42,7 @@ defmodule KafkaEx.Mixfile do main_deps = [ {:kayrock, "~> 0.1.12"}, {:credo, "~> 1.1", only: [:dev, :test], runtime: false}, - {:dialyxir, "~> 1.0.0-rc.3", only: :dev, runtime: false}, + {:dialyxir, "~> 1.0", only: :dev, runtime: false}, {:excoveralls, "~> 0.7", only: :test, runtime: false}, {:snappy, git: "https://github.com/fdmanana/snappy-erlang-nif", only: [:dev, :test]} diff --git a/mix.lock b/mix.lock index 84829ef0..5e43d144 100644 --- a/mix.lock +++ b/mix.lock @@ -4,9 +4,9 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "crc32cer": {:hex, :crc32cer, "0.1.10", "fb87abbf34b72f180f8c3a908cd1826c6cb9a59787d156a29e05de9e98be385e", [:rebar3], [], "hexpm", "5b1f47efd0a1b4b7411f1f35e14d3c8c6da6e6a2a725ec8f2cf1ab13703e5f38"}, "credo": {:hex, :credo, "1.1.5", "caec7a3cadd2e58609d7ee25b3931b129e739e070539ad1a0cd7efeeb47014f4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0bbd3222607ccaaac5c0340f7f525c627ae4d7aee6c8c8c108922620c5b6446"}, - "dialyxir": {:hex, :dialyxir, "1.0.0-rc.7", "6287f8f2cb45df8584317a4be1075b8c9b8a69de8eeb82b4d9e6c761cf2664cd", [:mix], [{:erlex, ">= 0.2.5", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "506294d6c543e4e5282d4852aead19ace8a35bedeb043f9256a06a6336827122"}, + "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "erlex": {:hex, :erlex, "0.2.5", "e51132f2f472e13d606d808f0574508eeea2030d487fc002b46ad97e738b0510", [:mix], [], "hexpm", "756d3e19b056339af674b715fdd752c5dac468cf9d0e2d1a03abf4574e99fbf8"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, "excoveralls": {:hex, :excoveralls, "0.12.1", "a553c59f6850d0aff3770e4729515762ba7c8e41eedde03208182a8dc9d0ce07", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "5c1f717066a299b1b732249e736c5da96bb4120d1e55dc2e6f442d251e18a812"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, diff --git a/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs b/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs index 5a696433..c69bd39d 100644 --- a/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs +++ b/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs @@ -65,40 +65,54 @@ defmodule KafkaEx.KayrockCompatibility0p10AndLaterTest do # note this checks against the version of broker we're running in test # api_key, max_version, min_version api_versions_kafka_0_11_0_1 = [ - [0, 3, 0], - [1, 5, 0], - [2, 2, 0], - [3, 4, 0], - [4, 0, 0], - [5, 0, 0], - [6, 3, 0], - [7, 1, 1], - [8, 3, 0], - [9, 3, 0], - [10, 1, 0], - [11, 2, 0], - [12, 1, 0], - [13, 1, 0], - [14, 1, 0], - [15, 1, 0], - [16, 1, 0], - [17, 0, 0], - [18, 1, 0], - [19, 2, 0], - [20, 1, 0], - [21, 0, 0], - [22, 0, 0], - [23, 0, 0], - [24, 0, 0], - [25, 0, 0], - [26, 0, 0], + [0, 8, 0], + [1, 11, 0], + [2, 5, 0], + [3, 9, 0], + [4, 4, 0], + [5, 2, 0], + [6, 6, 0], + [7, 3, 0], + [8, 8, 0], + [9, 7, 0], + [10, 3, 0], + [11, 7, 0], + [12, 4, 0], + [13, 4, 0], + [14, 5, 0], + [15, 5, 0], + [16, 3, 0], + [17, 1, 0], + [18, 3, 0], + [19, 5, 0], + [20, 4, 0], + [21, 1, 0], + [22, 3, 0], + [23, 3, 0], + [24, 1, 0], + [25, 1, 0], + [26, 1, 0], [27, 0, 0], - [28, 0, 0], - [29, 0, 0], - [30, 0, 0], - [31, 0, 0], - [32, 0, 0], - [33, 0, 0] + [28, 3, 0], + [29, 2, 0], + [30, 2, 0], + [31, 2, 0], + [32, 2, 0], + [33, 1, 0], + [34, 1, 0], + [35, 1, 0], + [36, 2, 0], + [37, 2, 0], + [38, 2, 0], + [39, 2, 0], + [40, 2, 0], + [41, 2, 0], + [42, 2, 0], + [43, 2, 0], + [44, 1, 0], + [45, 0, 0], + [46, 0, 0], + [47, 0, 0] ] response = KafkaEx.api_versions(worker_name: client) diff --git a/test/integration/server0_p_10_p_1_test.exs b/test/integration/server0_p_10_p_1_test.exs index 4285c9b5..fd78d507 100644 --- a/test/integration/server0_p_10_p_1_test.exs +++ b/test/integration/server0_p_10_p_1_test.exs @@ -11,40 +11,54 @@ defmodule KafkaEx.Server0P10P1.Test do # note this checks against the version of broker we're running in test # api_key, max_version, min_version api_versions_kafka_0_11_0_1 = [ - [0, 3, 0], - [1, 5, 0], - [2, 2, 0], - [3, 4, 0], - [4, 0, 0], - [5, 0, 0], - [6, 3, 0], - [7, 1, 1], - [8, 3, 0], - [9, 3, 0], - [10, 1, 0], - [11, 2, 0], - [12, 1, 0], - [13, 1, 0], - [14, 1, 0], - [15, 1, 0], - [16, 1, 0], - [17, 0, 0], - [18, 1, 0], - [19, 2, 0], - [20, 1, 0], - [21, 0, 0], - [22, 0, 0], - [23, 0, 0], - [24, 0, 0], - [25, 0, 0], - [26, 0, 0], + [0, 8, 0], + [1, 11, 0], + [2, 5, 0], + [3, 9, 0], + [4, 4, 0], + [5, 2, 0], + [6, 6, 0], + [7, 3, 0], + [8, 8, 0], + [9, 7, 0], + [10, 3, 0], + [11, 7, 0], + [12, 4, 0], + [13, 4, 0], + [14, 5, 0], + [15, 5, 0], + [16, 3, 0], + [17, 1, 0], + [18, 3, 0], + [19, 5, 0], + [20, 4, 0], + [21, 1, 0], + [22, 3, 0], + [23, 3, 0], + [24, 1, 0], + [25, 1, 0], + [26, 1, 0], [27, 0, 0], - [28, 0, 0], - [29, 0, 0], - [30, 0, 0], - [31, 0, 0], - [32, 0, 0], - [33, 0, 0] + [28, 3, 0], + [29, 2, 0], + [30, 2, 0], + [31, 2, 0], + [32, 2, 0], + [33, 1, 0], + [34, 1, 0], + [35, 1, 0], + [36, 2, 0], + [37, 2, 0], + [38, 2, 0], + [39, 2, 0], + [40, 2, 0], + [41, 2, 0], + [42, 2, 0], + [43, 2, 0], + [44, 1, 0], + [45, 0, 0], + [46, 0, 0], + [47, 0, 0] ] response = KafkaEx.api_versions() diff --git a/test/socket_test.exs b/test/socket_test.exs index 9a71fc20..47a69fe2 100644 --- a/test/socket_test.exs +++ b/test/socket_test.exs @@ -36,6 +36,7 @@ defmodule KafkaEx.Socket.Test do {:ok, listen_socket} = :ssl.listen(port, [ :binary, + {:verify, :verify_none}, {:active, false}, {:reuseaddr, true}, {:packet, 0}, @@ -49,7 +50,7 @@ defmodule KafkaEx.Socket.Test do defp listen(socket) do case :ssl.transport_accept(socket) do {:ok, conn} -> - :ok = :ssl.ssl_accept(conn) + {:ok, _socket} = :ssl.handshake(conn) pid = spawn_link(fn -> recv(conn) end) :ssl.controlling_process(socket, pid) @@ -73,92 +74,103 @@ defmodule KafkaEx.Socket.Test do setup_all do :ssl.start() - SSLServer.start(3030) - Server.start(3040) - {:ok, [ssl_port: 3030, port: 3040]} end - test "create a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}], - false - ) - - assert socket.ssl == false - KafkaEx.Socket.close(socket) - end + describe "without SSL socket" do + setup do + Server.start(3040) + {:ok, [port: 3040]} + end - test "send and receive using a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}, {:active, false}], - false - ) - - KafkaEx.Socket.send(socket, 'ping') - assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) - KafkaEx.Socket.close(socket) - end + test "create a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}], + false + ) + + assert socket.ssl == false + KafkaEx.Socket.close(socket) + end - test "retrieve info from a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}, {:active, false}], - false - ) - - info = KafkaEx.Socket.info(socket) - assert info[:name] == 'tcp_inet' - KafkaEx.Socket.close(socket) - assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') - end + test "send and receive using a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}, {:active, false}], + false + ) + + KafkaEx.Socket.send(socket, 'ping') + assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) + KafkaEx.Socket.close(socket) + end - test "create a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}], - true - ) - - assert socket.ssl == true - KafkaEx.Socket.close(socket) + test "retrieve info from a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}, {:active, false}], + false + ) + + info = KafkaEx.Socket.info(socket) + assert info[:name] == 'tcp_inet' + KafkaEx.Socket.close(socket) + assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + end end - test "send and receive using a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}, {:active, false}], - true - ) - - KafkaEx.Socket.send(socket, 'ping') - assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) - KafkaEx.Socket.close(socket) - end + describe "with ssl socket" do + setup do + SSLServer.start(3030) + {:ok, [ssl_port: 3030]} + end + + test "create a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}], + true + ) + + assert socket.ssl == true + KafkaEx.Socket.close(socket) + end - test "retrieve info from a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}, {:active, false}], - true - ) - - info = KafkaEx.Socket.info(socket) - assert info[:name] == 'tcp_inet' - KafkaEx.Socket.close(socket) - assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + test "send and receive using a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}, {:active, false}], + true + ) + + KafkaEx.Socket.send(socket, 'ping') + assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) + KafkaEx.Socket.close(socket) + end + + test "retrieve info from a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}, {:active, false}], + true + ) + + info = KafkaEx.Socket.info(socket) + assert info[:name] == 'tcp_inet' + KafkaEx.Socket.close(socket) + assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + end end end From d598aacf5fb9291a0d5a57504497ae81a49f27d0 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 21 Apr 2022 16:36:16 +0200 Subject: [PATCH 25/30] MIx format & fix dialyxir --- .github/workflows/checks.yml | 2 ++ lib/kafka_ex.ex | 5 +++-- lib/kafka_ex/protocol/fetch.ex | 2 +- lib/kafka_ex/protocol/produce.ex | 2 +- lib/kafka_ex/server.ex | 10 +++++----- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d909801c..be96cd18 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -116,6 +116,8 @@ jobs: with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + - name: Retrieve Mix Dependencies Cache uses: actions/cache@v2 id: mix-cache # id to use in retrieve action diff --git a/lib/kafka_ex.ex b/lib/kafka_ex.ex index 6fcc927f..129aed47 100644 --- a/lib/kafka_ex.ex +++ b/lib/kafka_ex.ex @@ -297,11 +297,12 @@ defmodule KafkaEx do api_version = Keyword.get(opts, :api_version, 0) # same for offset_commit_api_version offset_commit_api_version = Keyword.get(opts, :offset_commit_api_version, 0) + # By default, it makes sense to synchronize the API version of the offset_commit and the offset_fetch # operations, otherwise we might commit the offsets in zookeeper and read them from Kafka, meaning # that the value would be incorrect. - offset_fetch_api_version = Keyword.get(opts, :offset_fetch_api_version, offset_commit_api_version) - + offset_fetch_api_version = + Keyword.get(opts, :offset_fetch_api_version, offset_commit_api_version) retrieved_offset = current_offset( diff --git a/lib/kafka_ex/protocol/fetch.ex b/lib/kafka_ex/protocol/fetch.ex index 7a0d7ba2..bfd12cac 100644 --- a/lib/kafka_ex/protocol/fetch.ex +++ b/lib/kafka_ex/protocol/fetch.ex @@ -67,7 +67,7 @@ defmodule KafkaEx.Protocol.Fetch do offset: integer, key: binary, value: binary, - headers: [{key::binary, value::binary}], + headers: [{key :: binary, value :: binary}], topic: binary, partition: integer, # timestamp supported for `kafka_version: "kayrock"` ONLY diff --git a/lib/kafka_ex/protocol/produce.ex b/lib/kafka_ex/protocol/produce.ex index 92203469..ee2a82fe 100644 --- a/lib/kafka_ex/protocol/produce.ex +++ b/lib/kafka_ex/protocol/produce.ex @@ -59,7 +59,7 @@ defmodule KafkaEx.Protocol.Produce do @type t :: %Message{ key: binary, value: binary, - headers: [{key::binary, value::binary}], + headers: [{key :: binary, value :: binary}], timestamp: integer } end diff --git a/lib/kafka_ex/server.ex b/lib/kafka_ex/server.ex index 72236203..304a2fbc 100644 --- a/lib/kafka_ex/server.ex +++ b/lib/kafka_ex/server.ex @@ -582,11 +582,11 @@ defmodule KafkaEx.Server do state.api_versions ) - updated_state = %{ - state - | metadata: metadata, - correlation_id: correlation_id - } + updated_state = %{ + state + | metadata: metadata, + correlation_id: correlation_id + } {:reply, metadata, updated_state} end From d0bd9c63e08b59bef00e6bcbae2fd504eeb1add9 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 21 Apr 2022 16:51:26 +0200 Subject: [PATCH 26/30] Final fixes --- .github/workflows/checks.yml | 115 +++++++++++------------------------ .github/workflows/test.yml | 2 +- mix.exs | 2 + 3 files changed, 40 insertions(+), 79 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index be96cd18..15416b05 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -23,8 +23,10 @@ jobs: uses: styfle/cancel-workflow-action@0.9.0 with: access_token: ${{ github.token }} + - name: Checkout Github repo uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment uses: actions/setup-elixir@v1 with: @@ -32,32 +34,28 @@ jobs: otp-version: ${{matrix.erlang}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - - name: Retrieve Mix Dependencies Cache + - name: Retrieve Cached Dependencies uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action + id: mix-cache with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + path: | + deps + _build + priv/plts + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - name: Install Mix Dependencies + - name: Install Dependencies if: steps.mix-cache.outputs.cache-hit != 'true' run: | + mkdir -p priv/plts mix local.rebar --force mix local.hex --force mix deps.get + mix deps.compile + mix dialyzer --plt - - name: Compile Mix Dependencies - if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' - run: mix deps.compile - - run-checks: - name: check + static_code_analysis: + name: Static Code Analysis runs-on: ubuntu-20.04 needs: [dependencies] env: @@ -70,80 +68,41 @@ jobs: erlang: ['24.3'] steps: - - uses: actions/checkout@v2 - - name: Setup elixir & erlang enviroment - uses: actions/setup-elixir@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] - experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + access_token: ${{ github.token }} - - name: Retrieve Mix Dependencies Cache - uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action + - name: Checkout + uses: actions/checkout@v2 with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - - run: | - mix compile --force - mix credo - mix format --check-formatted + fetch-depth: 0 - dialyzer: - name: check / dialyzer - needs: [dependencies] - runs-on: ubuntu-latest - env: - MIX_ENV: dev - - strategy: - matrix: - elixir: ['1.12.3'] - erlang: ['24.3'] - - steps: - - uses: actions/checkout@v2 - - name: Setup elixir + - name: Setup elixir & erlang enviroment uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] otp-version: ${{matrix.erlang}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - - name: Retrieve Mix Dependencies Cache + - name: Retrieve Cached Dependencies uses: actions/cache@v2 - id: mix-cache # id to use in retrieve action + id: mix-cache with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + path: | + deps + _build + priv/plts + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} - - name: Retrieve Mix Dependencies Compilation Cache - uses: actions/cache@v2 - id: mix-deps-compile-cache # id to use in retrieve action - with: - path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Compile + run: mix compile --force - - name: Retrieve PLT Cache - uses: actions/cache@v1 - id: plt-cache - with: - path: priv/plts - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-plts-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Check Code Format + run: mix format --check-formatted - - name: Create PLTs - if: steps.plt-cache.outputs.cache-hit != 'true' - run: | - mkdir -p priv/plts - mix dialyzer --plt + - name: Run Credo + run: mix credo - - name: Run dialyzer - run: mix dialyzer --no-check --halt-exit-status + - name: Run Dialyzer + run: mix dialyzer --no-check \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d215b26..cc46392b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -98,4 +98,4 @@ jobs: - name: Docker ps run: docker ps -a - name: Run Tests - run: ./scripts/all_tests.sh + run: ./scripts/ci_tests.sh diff --git a/mix.exs b/mix.exs index b618b364..ff0e48d9 100644 --- a/mix.exs +++ b/mix.exs @@ -12,6 +12,8 @@ defmodule KafkaEx.Mixfile do elixir: "~> 1.6", dialyzer: [ plt_add_deps: :transitive, + plt_add_apps: [:ssl], + plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, flags: [ :error_handling, :race_conditions From 5a3b8613ef90fa827701ec91e3e5c75f1b26dfe3 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 5 May 2022 09:39:56 +0200 Subject: [PATCH 27/30] Add more otp & elixir versions to matrix Support multiple otp & elixir version for tests Support only newst version for static check. Handle backward compability in tests --- .github/workflows/checks.yml | 12 ++++++------ .github/workflows/test.yml | 35 ++++++++++++++++++++++------------- test/socket_test.exs | 12 +++++++++++- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 15416b05..0cf7af21 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,8 +15,8 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true strategy: matrix: - elixir: ['1.12.3'] - erlang: ['24.3'] + elixir: ['1.12'] + otp: ['24.3'] steps: - name: Cancel previous runs @@ -31,7 +31,7 @@ jobs: uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Cached Dependencies @@ -64,8 +64,8 @@ jobs: strategy: fail-fast: false matrix: - elixir: ['1.12.3'] - erlang: ['24.3'] + elixir: ['1.12'] + otp: ['24.3'] steps: - name: Cancel Previous Runs @@ -82,7 +82,7 @@ jobs: uses: actions/setup-elixir@v1 with: elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Cached Dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc46392b..14e57e72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,8 +14,13 @@ jobs: MIX_ENV: test strategy: matrix: - elixir: ['1.12.3'] - erlang: ['24.3'] + pair: + - elixir: 1.13 + otp: 24.3 + - elixir: 1.11 + otp: 21.3 + - elixir: 1.6 + otp: 20.3 steps: - name: Cancel previous runs @@ -27,8 +32,8 @@ jobs: - name: Setup elixir & erlang enviroment uses: actions/setup-elixir@v1 with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] + elixir-version: ${{matrix.pair.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.pair.otp}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Mix Dependencies Cache @@ -36,14 +41,14 @@ jobs: id: mix-cache # id to use in retrieve action with: path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - name: Retrieve Mix Dependencies Compilation Cache uses: actions/cache@v2 id: mix-deps-compile-cache # id to use in retrieve action with: path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - name: Install Mix Dependencies if: steps.mix-cache.outputs.cache-hit != 'true' run: | @@ -66,17 +71,21 @@ jobs: strategy: fail-fast: false matrix: - include: - - elixir: 1.12.3 - erlang: 24.3 + pair: + - elixir: 1.13 + otp: 24.3 + - elixir: 1.11 + otp: 21.3 + - elixir: 1.6 + otp: 20.3 steps: - uses: actions/checkout@v2 - name: Setup elixir & erlang enviroment uses: actions/setup-elixir@v1 with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.erlang}} # Define the OTP version [required] + elixir-version: ${{matrix.pair.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.pair.otp}} # Define the OTP version [required] experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 - name: Retrieve Mix Dependencies Cache @@ -84,14 +93,14 @@ jobs: id: mix-cache # id to use in retrieve action with: path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - name: Retrieve Mix Dependencies Compilation Cache uses: actions/cache@v2 id: mix-deps-compile-cache # id to use in retrieve action with: path: _build - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - name: Docker-compose up run: ./scripts/docker_up.sh diff --git a/test/socket_test.exs b/test/socket_test.exs index 47a69fe2..dab5e82a 100644 --- a/test/socket_test.exs +++ b/test/socket_test.exs @@ -50,7 +50,12 @@ defmodule KafkaEx.Socket.Test do defp listen(socket) do case :ssl.transport_accept(socket) do {:ok, conn} -> - {:ok, _socket} = :ssl.handshake(conn) + if opt_version_21_plus? do + {:ok, _socket} = :ssl.handshake(conn) + else + :ok = :ssl.ssl_accept(conn) + end + pid = spawn_link(fn -> recv(conn) end) :ssl.controlling_process(socket, pid) @@ -70,6 +75,11 @@ defmodule KafkaEx.Socket.Test do :ok end end + + defp opt_version_21_plus? do + {version, _} = System.otp_release() |> Float.parse() + version >= 21 + end end setup_all do From 5e038a14f89127dc2bbe054e78c0ceafc2d6712f Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 5 May 2022 10:43:11 +0200 Subject: [PATCH 28/30] Add badges --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d9f648f..45dc9c67 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ KafkaEx ======== -[![Build Status](https://travis-ci.org/kafkaex/kafka_ex.svg?branch=master)](https://travis-ci.org/kafkaex/kafka_ex) +[![CI Tests](https://github.com/kafkaex/kafka_ex/actions/workflows/test.yml/badge.svg)](https://github.com/kafkaex/kafka_ex/actions/workflows/test.yml) +[![CI Checks](https://github.com/kafkaex/kafka_ex/actions/workflows/checks.yml/badge.svg)](https://github.com/kafkaex/kafka_ex/actions/workflows/checks.yml) [![Coverage Status](https://coveralls.io/repos/github/kafkaex/kafka_ex/badge.svg?branch=master)](https://coveralls.io/github/kafkaex/kafka_ex?branch=master) [![Hex.pm version](https://img.shields.io/hexpm/v/kafka_ex.svg?style=flat-square)](https://hex.pm/packages/kafka_ex) [![Hex.pm downloads](https://img.shields.io/hexpm/dt/kafka_ex.svg?style=flat-square)](https://hex.pm/packages/kafka_ex) From 776da2057e66dd51465b500174785fa31f00ed79 Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 5 May 2022 10:48:00 +0200 Subject: [PATCH 29/30] Fix syntax in socket tests --- test/socket_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/socket_test.exs b/test/socket_test.exs index dab5e82a..b9bf6a58 100644 --- a/test/socket_test.exs +++ b/test/socket_test.exs @@ -50,7 +50,7 @@ defmodule KafkaEx.Socket.Test do defp listen(socket) do case :ssl.transport_accept(socket) do {:ok, conn} -> - if opt_version_21_plus? do + if opt_version_21_plus?() do {:ok, _socket} = :ssl.handshake(conn) else :ok = :ssl.ssl_accept(conn) From 158c0372f49173bb32a116cab0558318a0a0904f Mon Sep 17 00:00:00 2001 From: Argonus Date: Thu, 2 Jun 2022 10:53:06 +0200 Subject: [PATCH 30/30] Apply CR --- README.md | 2 +- test/integration/server0_p_10_p_1_test.exs | 4 ++-- test/socket_test.exs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 45dc9c67..c0c84d34 100644 --- a/README.md +++ b/README.md @@ -402,7 +402,7 @@ mix test --no-start If you are not using the Docker test cluster, you may need to modify `config/config.exs` for your set up. -The full test suite requires Kafka 0.10.1.0+. +The full test suite requires Kafka 2.1.0+. ##### Kafka >= 0.9.0 diff --git a/test/integration/server0_p_10_p_1_test.exs b/test/integration/server0_p_10_p_1_test.exs index fd78d507..63df475d 100644 --- a/test/integration/server0_p_10_p_1_test.exs +++ b/test/integration/server0_p_10_p_1_test.exs @@ -10,7 +10,7 @@ defmodule KafkaEx.Server0P10P1.Test do test "can retrieve api versions" do # note this checks against the version of broker we're running in test # api_key, max_version, min_version - api_versions_kafka_0_11_0_1 = [ + api_versions_kafka_2_1_0 = [ [0, 8, 0], [1, 11, 0], [2, 5, 0], @@ -69,7 +69,7 @@ defmodule KafkaEx.Server0P10P1.Test do throttle_time_ms: _ } = response - assert api_versions_kafka_0_11_0_1 == + assert api_versions_kafka_2_1_0 == api_versions |> Enum.map(&[&1.api_key, &1.max_version, &1.min_version]) end diff --git a/test/socket_test.exs b/test/socket_test.exs index b9bf6a58..b4ff1dfe 100644 --- a/test/socket_test.exs +++ b/test/socket_test.exs @@ -50,7 +50,7 @@ defmodule KafkaEx.Socket.Test do defp listen(socket) do case :ssl.transport_accept(socket) do {:ok, conn} -> - if opt_version_21_plus?() do + if otp_version_21_plus?() do {:ok, _socket} = :ssl.handshake(conn) else :ok = :ssl.ssl_accept(conn) @@ -76,7 +76,7 @@ defmodule KafkaEx.Socket.Test do end end - defp opt_version_21_plus? do + defp otp_version_21_plus? do {version, _} = System.otp_release() |> Float.parse() version >= 21 end