Add first integration test

Add integration tests based on tests containers library.
This will allow us to test more complex scenarios than a single docker-compose based ones.

s

s

.github/workflows/integration.yml

@@ -0,0 +1,93 @@
+name: CI Integration
+
+on:
+  pull_request: []
+
+jobs:
+  dependencies:
+    name: integration | setup dependencies
+    runs-on: ubuntu-20.04
+    env:
+      MIX_ENV: test
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    strategy:
+      matrix:
+        elixir: ['1.15']
+        otp: ['26.1']
+
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/cancel-workflow-action@0.9.0
+        with:
+          access_token: ${{ github.token }}
+
+      - name: Checkout Github repo
+        uses: actions/checkout@v2
+
+      - name: Setup elixir & erlang environment
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{matrix.elixir}} # Define the elixir version [required]
+          otp-version: ${{matrix.otp}} # Define the OTP version [required]
+          experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31
+
+      - name: Retrieve Cached Dependencies
+        uses: actions/cache@v2
+        id: mix-cache
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }}
+
+      - name: Install Dependencies
+        if: steps.mix-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p priv/plts
+          mix local.rebar --force
+          mix local.hex --force
+          mix deps.get
+          mix deps.compile
+
+  integration_test:
+    name: Integration Test
+    runs-on: ubuntu-20.04
+    needs: [dependencies]
+    env:
+      MIX_ENV: test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        elixir: ['1.15']
+        otp: ['26.1']
+
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.6.0
+        with:
+          access_token: ${{ github.token }}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup elixir & erlang environment
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{matrix.elixir}} # Define the elixir version [required]
+          otp-version: ${{matrix.otp}} # Define the OTP version [required]
+          experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31
+
+      - name: Retrieve Cached Dependencies
+        uses: actions/cache@v2
+        id: mix-cache
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }}
+
+      - name: Run Test
+        run: mix test.integration

config/test.exs

@@ -0,0 +1 @@
+config :testcontainers, enabled: true

mix.exs

@@ -10,9 +10,10 @@ defmodule Kayrock.MixProject do
       elixir: "~> 1.10",
       elixirc_paths: elixirc_paths(Mix.env()),
       test_coverage: [tool: ExCoveralls],
-      preferred_cli_env: [coveralls: :test],
+      preferred_cli_env: [coveralls: :test, "test.integration": :test],
       start_permanent: Mix.env() == :prod,
       deps: deps(),
+      aliases: aliases(),
       dialyzer: [
         plt_add_apps: [:mix],
         flags: [:error_handling, :race_conditions]
@@ -44,8 +45,6 @@ defmodule Kayrock.MixProject do
       {:crc32cer, "~> 0.1"},
       {:varint, "~> 1.2"},
       {:connection, "~> 1.1"},
-      # Integration Tests
-      {:testcontainers, "~> 1.5", only: [:test]},
 
       # Dev/Test
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
@@ -56,6 +55,15 @@ defmodule Kayrock.MixProject do
       {:snappy, git: "https://github.com/fdmanana/snappy-erlang-nif", only: [:dev, :test]},
       {:snappyer, "~> 1.2", only: [:dev, :test]}
     ]
+    |> integration_test_deps()
+  end
+
+  defp integration_test_deps(deps_list) do
+    if Version.match?(System.version(), ">= 1.15.0") do
+      [{:testcontainers, "~> 1.5"} | deps_list]
+    else
+      deps_list
+    end
   end
 
   defp elixirc_paths(:test), do: ["lib", "test/support"]
@@ -70,4 +78,10 @@ defmodule Kayrock.MixProject do
       links: %{"GitHub" => @source_url}
     ]
   end
+
+  defp aliases do
+    [
+      "test.integration": "test --only integration_v2"
+    ]
+  end
 end

test/integration/topic_management_test.exs

@@ -0,0 +1,117 @@
+defmodule Kayrock.Integration.TopicManagementTest do
+  use ExUnit.Case, async: true
+  use Kayrock.IntegrationCase
+  import Kayrock.TestSupport
+  import Kayrock.Convenience
+
+  alias Testcontainers.Container
+  alias Testcontainers.KafkaContainer
+
+  container(:kafka, KafkaContainer.new(), shared: true)
+
+  @moduletag :integration_v2
+  describe "topic management API" do
+    for version <- [0, 1, 2] do
+      test "v#{version} - allows to manage topic", %{kafka: kafka} do
+        uris = [{"localhost", Container.mapped_port(kafka, 9092)}]
+        api_version = unquote(version)
+        {:ok, client_pid} = Kayrock.Client.start_link(uris)
+        topic_name = unique_string()
+
+        # Get Topics
+        refute topic_exists?(client_pid, topic_name)
+
+        # Creates Topic
+        create_request = create_topic_request(topic_name, api_version)
+        {:ok, _} = Kayrock.client_call(client_pid, create_request, :controller)
+
+        # Get Topic
+        topic = get_topic_metadata(client_pid, topic_name)
+        assert topic.topic == topic_name
+        assert length(topic.partition_metadata) == 3
+
+        # Create Partitions
+        create_partition_config = create_topic_partition(topic_name, api_version)
+        {:ok, res} = Kayrock.client_call(client_pid, create_partition_config, :controller)
+        assert List.first(res.topic_errors).error_code == 0
+
+        # Get Updated Topic
+        topic = get_topic_metadata(client_pid, topic_name)
+        assert length(topic.partition_metadata) == 5
+
+        # Update Topic Config
+        alter_config = alter_topic_config(topic_name, api_version)
+        {:ok, res} = Kayrock.client_call(client_pid, alter_config, :controller)
+        assert List.first(res.resources).error_code == 0
+
+        # Get Topic Config
+        describe_config = describe_config(topic_name, api_version)
+        {:ok, res} = Kayrock.client_call(client_pid, describe_config, :controller)
+        resource = List.first(res.resources)
+        assert resource.error_code == 0
+        config = List.first(resource.config_entries)
+        assert config.config_name == "cleanup.policy"
+        assert config.config_value == "compact"
+
+        # Deletes Topic
+        max_version = min(Kayrock.DeleteTopics.max_vsn(), api_version)
+        {:ok, _} = Kayrock.delete_topics(client_pid, [topic_name], 1000, max_version)
+
+        # Get Topic
+        refute topic_exists?(client_pid, topic_name)
+      end
+    end
+  end
+
+  # Helpers
+  defp create_topic_request(topic_name, api_version) do
+    api_version = min(Kayrock.CreateTopics.max_vsn(), api_version)
+    request = Kayrock.CreateTopics.get_request_struct(api_version)
+
+    topic_config = %{
+      topic: topic_name,
+      num_partitions: 3,
+      replication_factor: 1,
+      replica_assignment: [],
+      config_entries: []
+    }
+
+    %{request | create_topic_requests: [topic_config], timeout: 1000}
+  end
+
+  defp create_topic_partition(topic_name, api_version) do
+    api_version = min(Kayrock.CreatePartitions.max_vsn(), api_version)
+    request = Kayrock.CreatePartitions.get_request_struct(api_version)
+    partition_config = %{topic: topic_name, new_partitions: %{count: 5, assignment: nil}}
+    %{request | topic_partitions: [partition_config], timeout: 1000, validate_only: false}
+  end
+
+  defp alter_topic_config(topic_name, api_version) do
+    api_version = min(Kayrock.AlterConfigs.max_vsn(), api_version)
+    request = Kayrock.AlterConfigs.get_request_struct(api_version)
+    config = %{config_name: "cleanup.policy", config_value: "compact"}
+
+    %{
+      request
+      | resources: [%{resource_type: 2, resource_name: topic_name, config_entries: [config]}],
+        validate_only: false
+    }
+  end
+
+  defp describe_config(topic_name, api_version) do
+    api_version = min(Kayrock.DescribeConfigs.max_vsn(), api_version)
+    request = Kayrock.DescribeConfigs.get_request_struct(api_version)
+
+    %{
+      request
+      | resources: [
+          %{resource_type: 2, resource_name: topic_name, config_names: ["cleanup.policy"]}
+        ]
+    }
+  end
+
+  def get_topic_metadata(pid, topic) when is_pid(pid) and is_binary(topic) do
+    {:ok, [topic]} = Kayrock.topics_metadata(pid, [topic])
+    topic
+  end
+end

test/support/integration_case.ex

@@ -0,0 +1,23 @@
+defmodule Kayrock.IntegrationCase do
+  @moduledoc """
+  Testcontainer integration case template
+  """
+  use ExUnit.CaseTemplate
+
+  if Code.ensure_compiled?(Testcontainers) do
+    using do
+      quote do
+        @moduletag :integration_v2
+        import Testcontainers.ExUnit
+      end
+    end
+
+    setup_all do
+      if Code.ensure_compiled?(Testcontainers) do
+        {:ok, _pid} = Testcontainers.start_link()
+      end
+
+      :ok
+    end
+  end
+end

test/support/test_support.ex

@@ -1,5 +1,13 @@
 defmodule Kayrock.TestSupport do
   @moduledoc "Support code for tests"
+
+  @doc """
+  Returns a unique string for use in tests.
+  """
+  def unique_string do
+    "test-topic-#{:erlang.unique_integer([:positive])}"
+  end
+
   def compare_binaries(lhs, rhs) do
     bytes_per_chunk = 16
     chunks_lhs = chunk_binary(lhs, bytes_per_chunk)

test/test_helper.exs

@@ -1,2 +1,2 @@
-ExUnit.configure(exclude: :integration)
+ExUnit.configure(exclude: [:integration, :integration_v2])
 ExUnit.start()