From baf6484037618b77ab1ada6982641342a9f0053d Mon Sep 17 00:00:00 2001 From: Argonus Date: Sat, 2 Dec 2023 15:53:00 +0100 Subject: [PATCH] Add first integration test Add integration tests based on tests containers library. This will allow us to test more complex scenarios than a single docker-compose based ones. s --- .github/workflows/integration.yml | 93 +++++++++++++++++ config/test.exs | 1 + mix.exs | 20 +++- test/integration/topic_management_test.exs | 114 +++++++++++++++++++++ test/kayrock/client/create_topic_test.exs | 46 --------- test/support/integration_case.ex | 47 +++++++++ test/support/test_support.ex | 8 ++ test/test_helper.exs | 2 +- 8 files changed, 281 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 config/test.exs create mode 100644 test/integration/topic_management_test.exs delete mode 100644 test/kayrock/client/create_topic_test.exs create mode 100644 test/support/integration_case.ex diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..b236419 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,93 @@ +name: CI Integration + +on: + pull_request: [] + +jobs: + dependencies: + name: integration | setup dependencies + runs-on: ubuntu-20.04 + env: + MIX_ENV: test + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + strategy: + matrix: + elixir: ['1.15'] + otp: ['26.1'] + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + + - name: Checkout Github repo + uses: actions/checkout@v2 + + - name: Setup elixir & erlang environment + uses: erlef/setup-beam@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Cached Dependencies + uses: actions/cache@v2 + id: mix-cache + with: + path: | + deps + _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} + + - name: Install Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix local.rebar --force + mix local.hex --force + mix deps.get + mix deps.compile + + integration_test: + name: Integration Test + runs-on: ubuntu-20.04 + needs: [dependencies] + env: + MIX_ENV: test + + strategy: + fail-fast: false + matrix: + elixir: ['1.15'] + otp: ['26.1'] + + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ github.token }} + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup elixir & erlang environment + uses: erlef/setup-beam@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Cached Dependencies + uses: actions/cache@v2 + id: mix-cache + with: + path: | + deps + _build + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} + + - name: Run Test + run: mix test.integration diff --git a/config/test.exs b/config/test.exs new file mode 100644 index 0000000..ef657aa --- /dev/null +++ b/config/test.exs @@ -0,0 +1 @@ +config :testcontainers, enabled: true diff --git a/mix.exs b/mix.exs index 0ccdb53..6b80f0e 100644 --- a/mix.exs +++ b/mix.exs @@ -10,9 +10,10 @@ defmodule Kayrock.MixProject do elixir: "~> 1.10", elixirc_paths: elixirc_paths(Mix.env()), test_coverage: [tool: ExCoveralls], - preferred_cli_env: [coveralls: :test], + preferred_cli_env: [coveralls: :test, "test.integration": :test], start_permanent: Mix.env() == :prod, deps: deps(), + aliases: aliases(), dialyzer: [ plt_add_apps: [:mix], flags: [:error_handling] @@ -44,8 +45,6 @@ defmodule Kayrock.MixProject do {:crc32cer, "~> 0.1"}, {:varint, "~> 1.2"}, {:connection, "~> 1.1"}, - # Integration Tests - {:testcontainers, "~> 1.5", only: [:test]}, # Dev/Test {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, @@ -56,6 +55,15 @@ defmodule Kayrock.MixProject do {:snappy, git: "https://github.com/fdmanana/snappy-erlang-nif", only: [:dev, :test]}, {:snappyer, "~> 1.2", only: [:dev, :test]} ] + |> integration_test_deps() + end + + defp integration_test_deps(deps_list) do + if Version.match?(System.version(), ">= 1.15.0") do + [{:testcontainers, "~> 1.5"} | deps_list] + else + deps_list + end end defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -70,4 +78,10 @@ defmodule Kayrock.MixProject do links: %{"GitHub" => @source_url} ] end + + defp aliases do + [ + "test.integration": "test --only integration_v2" + ] + end end diff --git a/test/integration/topic_management_test.exs b/test/integration/topic_management_test.exs new file mode 100644 index 0000000..d2aa362 --- /dev/null +++ b/test/integration/topic_management_test.exs @@ -0,0 +1,114 @@ +defmodule Kayrock.Integration.TopicManagementTest do + use Kayrock.IntegrationCase + use ExUnit.Case, async: true + + import Kayrock.TestSupport + import Kayrock.Convenience + + container(:kafka, KafkaContainer.new(), shared: true) + + describe "topic management API" do + for version <- [0, 1, 2] do + test "v#{version} - allows to manage topic", %{kafka: kafka} do + uris = [{"localhost", Container.mapped_port(kafka, 9092)}] + api_version = unquote(version) + {:ok, client_pid} = Kayrock.Client.start_link(uris) + topic_name = unique_string() + + # Get Topics + refute topic_exists?(client_pid, topic_name) + + # Creates Topic + create_request = create_topic_request(topic_name, api_version) + {:ok, _} = Kayrock.client_call(client_pid, create_request, :controller) + + # Get Topic + topic = get_topic_metadata(client_pid, topic_name) + assert topic.topic == topic_name + assert length(topic.partition_metadata) == 3 + + # Create Partitions + create_partition_config = create_topic_partition(topic_name, api_version) + {:ok, res} = Kayrock.client_call(client_pid, create_partition_config, :controller) + assert List.first(res.topic_errors).error_code == 0 + + # Get Updated Topic + topic = get_topic_metadata(client_pid, topic_name) + assert length(topic.partition_metadata) == 5 + + # Update Topic Config + alter_config = alter_topic_config(topic_name, api_version) + {:ok, res} = Kayrock.client_call(client_pid, alter_config, :controller) + assert List.first(res.resources).error_code == 0 + + # Get Topic Config + describe_config = describe_config(topic_name, api_version) + {:ok, res} = Kayrock.client_call(client_pid, describe_config, :controller) + resource = List.first(res.resources) + assert resource.error_code == 0 + config = List.first(resource.config_entries) + assert config.config_name == "cleanup.policy" + assert config.config_value == "compact" + + # Deletes Topic + max_version = min(Kayrock.DeleteTopics.max_vsn(), api_version) + {:ok, _} = Kayrock.delete_topics(client_pid, [topic_name], 1000, max_version) + + # Get Topic + refute topic_exists?(client_pid, topic_name) + end + end + end + + # Helpers + defp create_topic_request(topic_name, api_version) do + api_version = min(Kayrock.CreateTopics.max_vsn(), api_version) + request = Kayrock.CreateTopics.get_request_struct(api_version) + + topic_config = %{ + topic: topic_name, + num_partitions: 3, + replication_factor: 1, + replica_assignment: [], + config_entries: [] + } + + %{request | create_topic_requests: [topic_config], timeout: 1000} + end + + defp create_topic_partition(topic_name, api_version) do + api_version = min(Kayrock.CreatePartitions.max_vsn(), api_version) + request = Kayrock.CreatePartitions.get_request_struct(api_version) + partition_config = %{topic: topic_name, new_partitions: %{count: 5, assignment: nil}} + %{request | topic_partitions: [partition_config], timeout: 1000, validate_only: false} + end + + defp alter_topic_config(topic_name, api_version) do + api_version = min(Kayrock.AlterConfigs.max_vsn(), api_version) + request = Kayrock.AlterConfigs.get_request_struct(api_version) + config = %{config_name: "cleanup.policy", config_value: "compact"} + + %{ + request + | resources: [%{resource_type: 2, resource_name: topic_name, config_entries: [config]}], + validate_only: false + } + end + + defp describe_config(topic_name, api_version) do + api_version = min(Kayrock.DescribeConfigs.max_vsn(), api_version) + request = Kayrock.DescribeConfigs.get_request_struct(api_version) + + %{ + request + | resources: [ + %{resource_type: 2, resource_name: topic_name, config_names: ["cleanup.policy"]} + ] + } + end + + def get_topic_metadata(pid, topic) when is_pid(pid) and is_binary(topic) do + {:ok, [topic]} = Kayrock.topics_metadata(pid, [topic]) + topic + end +end diff --git a/test/kayrock/client/create_topic_test.exs b/test/kayrock/client/create_topic_test.exs deleted file mode 100644 index 0fa6e6e..0000000 --- a/test/kayrock/client/create_topic_test.exs +++ /dev/null @@ -1,46 +0,0 @@ -defmodule Kayrock.Client.CreateTopicTest do - # CreateTopics Request (Version: 3) => [create_topic_requests] timeout validate_only - # create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries] - # topic => STRING - # num_partitions => INT32 - # replication_factor => INT16 - # replica_assignment => partition [replicas] - # partition => INT32 - # replicas => INT32 - # config_entries => config_name config_value - # config_name => STRING - # config_value => NULLABLE_STRING - # timeout => INT32 - # validate_only => BOOLEAN - - use Kayrock.ClientCase - - alias Kayrock.ErrorCode - - import Kayrock.Convenience - - test "delete topic - topic doesn't exist", %{client: client} do - topic = unique_topic() - refute topic_exists?(client, topic) - - {:ok, resp} = Kayrock.delete_topics(client, [topic]) - [topic_error_code] = resp.topic_error_codes - assert topic_error_code[:error_code] == ErrorCode.unknown_topic() - end - - test "create a topic - specify num partitions", %{client: client} do - topic = unique_topic() - refute topic_exists?(client, topic) - - {:ok, _} = - Kayrock.create_topics( - client, - [%{topic: topic, num_partitions: 4, replication_factor: 2}], - 1000 - ) - - assert topic_exists?(client, topic) - - {:ok, _} = Kayrock.delete_topics(client, [topic], 1000) - end -end diff --git a/test/support/integration_case.ex b/test/support/integration_case.ex new file mode 100644 index 0000000..cc65095 --- /dev/null +++ b/test/support/integration_case.ex @@ -0,0 +1,47 @@ +defmodule Kayrock.IntegrationCase do + @moduledoc """ + Testcontainer integration case template + """ + use ExUnit.CaseTemplate + + if Code.ensure_compiled?(Testcontainers) do + using do + quote do + @moduletag :integration_v2 + import Testcontainers.ExUnit + + alias Testcontainers.Container + alias Testcontainers.KafkaContainer + end + end + + setup_all do + {:ok, _pid} = Testcontainers.start_link() + :ok + end + else + defmodule TestcontainersStub do + @moduledoc false + + def container(_name, _config, _opts) do + :ok + end + end + + defmodule KafkaContainerStub do + @moduledoc false + + def new() do + end + end + + using do + quote do + @moduletag :integration_v2 + import TestcontainersStub + + alias Kayrock.IntegrationCase.KafkaContainerStub, as: KafkaContainer + end + end + end +end diff --git a/test/support/test_support.ex b/test/support/test_support.ex index c842f20..7f19ecb 100644 --- a/test/support/test_support.ex +++ b/test/support/test_support.ex @@ -1,5 +1,13 @@ defmodule Kayrock.TestSupport do @moduledoc "Support code for tests" + + @doc """ + Returns a unique string for use in tests. + """ + def unique_string do + "test-topic-#{:erlang.unique_integer([:positive])}" + end + def compare_binaries(lhs, rhs) do bytes_per_chunk = 16 chunks_lhs = chunk_binary(lhs, bytes_per_chunk) diff --git a/test/test_helper.exs b/test/test_helper.exs index 2a45934..b13773e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,2 @@ -ExUnit.configure(exclude: :integration) +ExUnit.configure(exclude: [:integration, :integration_v2]) ExUnit.start()