Skip to content

Commit

Permalink
chore: update messages and fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
oliveigah committed Mar 28, 2024
1 parent 489556a commit 6b14584
Show file tree
Hide file tree
Showing 31 changed files with 1,822 additions and 88 deletions.
20 changes: 19 additions & 1 deletion lib/klife_protocol/generated/messages/add_offsets_to_txn.ex
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ defmodule KlifeProtocol.Messages.AddOffsetsToTxn do
- Version 1 is the same as version 0.
- Version 2 adds the support for new error code PRODUCER_FENCED.
- Version 3 enables flexible versions.
- Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
Response versions summary:
- Starting in version 1, on quota violation brokers send out responses before throttling.
- Version 2 adds the support for new error code PRODUCER_FENCED.
- Version 3 enables flexible versions.
- Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
"""

Expand Down Expand Up @@ -85,7 +87,7 @@ defmodule KlifeProtocol.Messages.AddOffsetsToTxn do
@doc """
Returns the current max supported version of this message.
"""
def max_supported_version(), do: 3
def max_supported_version(), do: 4

@doc """
Returns the current min supported version of this message.
Expand Down Expand Up @@ -131,6 +133,15 @@ defmodule KlifeProtocol.Messages.AddOffsetsToTxn do
tag_buffer: {:tag_buffer, []}
]

defp request_schema(4),
do: [
transactional_id: {:compact_string, %{is_nullable?: false}},
producer_id: {:int64, %{is_nullable?: false}},
producer_epoch: {:int16, %{is_nullable?: false}},
group_id: {:compact_string, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]

defp request_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AddOffsetsToTxn")

Expand Down Expand Up @@ -159,6 +170,13 @@ defmodule KlifeProtocol.Messages.AddOffsetsToTxn do
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(4),
do: [
throttle_time_ms: {:int32, %{is_nullable?: false}},
error_code: {:int16, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AddOffsetsToTxn")
end
51 changes: 50 additions & 1 deletion lib/klife_protocol/generated/messages/add_partitions_to_txn.ex
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ defmodule KlifeProtocol.Messages.AddPartitionsToTxn do
- Version 2 adds the support for new error code PRODUCER_FENCED.
- Version 3 enables flexible versions.
- Version 4 adds VerifyOnly field to check if partitions are already in transaction and adds support to batch multiple transactions.
- Version 5 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
Versions 3 and below will be exclusively used by clients and versions 4 and above will be used by brokers.
Response versions summary:
- Starting in version 1, on quota violation brokers send out responses before throttling.
- Version 2 adds the support for new error code PRODUCER_FENCED.
- Version 3 enables flexible versions.
- Version 4 adds support to batch multiple transactions and a top level error code.
- Version 5 adds support for new error code TRANSACTION_ABORTABLE (KIP-890).
"""

Expand Down Expand Up @@ -106,7 +108,7 @@ defmodule KlifeProtocol.Messages.AddPartitionsToTxn do
@doc """
Returns the current max supported version of this message.
"""
def max_supported_version(), do: 4
def max_supported_version(), do: 5

@doc """
Returns the current min supported version of this message.
Expand Down Expand Up @@ -194,6 +196,27 @@ defmodule KlifeProtocol.Messages.AddPartitionsToTxn do
tag_buffer: {:tag_buffer, []}
]

defp request_schema(5),
do: [
transactions:
{{:compact_array,
[
transactional_id: {:compact_string, %{is_nullable?: false}},
producer_id: {:int64, %{is_nullable?: false}},
producer_epoch: {:int16, %{is_nullable?: false}},
verify_only: {:boolean, %{is_nullable?: false}},
topics:
{{:compact_array,
[
name: {:compact_string, %{is_nullable?: false}},
partitions: {{:compact_array, :int32}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]

defp request_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AddPartitionsToTxn")

Expand Down Expand Up @@ -290,6 +313,32 @@ defmodule KlifeProtocol.Messages.AddPartitionsToTxn do
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(5),
do: [
throttle_time_ms: {:int32, %{is_nullable?: false}},
error_code: {:int16, %{is_nullable?: false}},
results_by_transaction:
{{:compact_array,
[
transactional_id: {:compact_string, %{is_nullable?: false}},
topic_results:
{{:compact_array,
[
name: {:compact_string, %{is_nullable?: false}},
results_by_partition:
{{:compact_array,
[
partition_index: {:int32, %{is_nullable?: false}},
partition_error_code: {:int16, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AddPartitionsToTxn")
end
2 changes: 1 addition & 1 deletion lib/klife_protocol/generated/messages/alter_partition.ex
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ defmodule KlifeProtocol.Messages.AlterPartition do
- Version 1 adds LeaderRecoveryState field (KIP-704).
- Version 2 adds TopicId field to replace TopicName field, can return the following new errors:
INELIGIBLE_REPLICA, NEW_LEADER_ELECTED and UNKNOWN_TOPIC_ID (KIP-841).
- Version 3 is the same as vesion 2 (KIP-903).
- Version 3 is the same as version 2 (KIP-903).
"""

Expand Down
162 changes: 162 additions & 0 deletions lib/klife_protocol/generated/messages/assign_replicas_to_dirs.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
# DO NOT EDIT THIS FILE MANUALLY
# This module is automatically generated by running mix task generate_file
# every change must be done inside the mix task directly

defmodule KlifeProtocol.Messages.AssignReplicasToDirs do
@moduledoc """
Kafka protocol AssignReplicasToDirs message
Request versions summary:
Response versions summary:
"""

alias KlifeProtocol.Deserializer
alias KlifeProtocol.Serializer
alias KlifeProtocol.Header

@api_key 73
@min_flexible_version_req 0
@min_flexible_version_res 0

@doc """
Receives a map and serialize it to kafka wire format of the given version.
Input content fields:
- broker_id: The ID of the requesting broker (int32 | versions 0+)
- broker_epoch: The epoch of the requesting broker (int64 | versions 0+)
- directories: ([]DirectoryData | versions 0+)
- id: The ID of the directory (uuid | versions 0+)
- topics: ([]TopicData | versions 0+)
- topic_id: The ID of the assigned topic (uuid | versions 0+)
- partitions: ([]PartitionData | versions 0+)
- partition_index: The partition index (int32 | versions 0+)
"""
def serialize_request(%{headers: headers, content: content}, version) do
headers
|> Map.put(:request_api_key, @api_key)
|> Map.put(:request_api_version, version)
|> Header.serialize_request(req_header_version(version))
|> then(&Serializer.execute(content, request_schema(version), &1))
end

@doc """
Receive a binary in the kafka wire format and deserialize it into a map.
Response content fields:
- throttle_time_ms: The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota. (int32 | versions 0+)
- error_code: The top level response error code (int16 | versions 0+)
- directories: ([]DirectoryData | versions 0+)
- id: The ID of the directory (uuid | versions 0+)
- topics: ([]TopicData | versions 0+)
- topic_id: The ID of the assigned topic (uuid | versions 0+)
- partitions: ([]PartitionData | versions 0+)
- partition_index: The partition index (int32 | versions 0+)
- error_code: The partition level error code (int16 | versions 0+)
"""
def deserialize_response(data, version, with_header? \\ true)

def deserialize_response(data, version, true) do
{:ok, {headers, rest_data}} = Header.deserialize_response(data, res_header_version(version))

case Deserializer.execute(rest_data, response_schema(version)) do
{:ok, {content, <<>>}} ->
{:ok, %{headers: headers, content: content}}

{:error, _reason} = err ->
err
end
end

def deserialize_response(data, version, false) do
case Deserializer.execute(data, response_schema(version)) do
{:ok, {content, <<>>}} ->
{:ok, %{content: content}}

{:error, _reason} = err ->
err
end
end

@doc """
Returns the message api key number.
"""
def api_key(), do: @api_key

@doc """
Returns the current max supported version of this message.
"""
def max_supported_version(), do: 0

@doc """
Returns the current min supported version of this message.
"""
def min_supported_version(), do: 0

defp req_header_version(msg_version),
do: if(msg_version >= @min_flexible_version_req, do: 2, else: 1)

defp res_header_version(msg_version),
do: if(msg_version >= @min_flexible_version_res, do: 1, else: 0)

defp request_schema(0),
do: [
broker_id: {:int32, %{is_nullable?: false}},
broker_epoch: {:int64, %{is_nullable?: false}},
directories:
{{:compact_array,
[
id: {:uuid, %{is_nullable?: false}},
topics:
{{:compact_array,
[
topic_id: {:uuid, %{is_nullable?: false}},
partitions:
{{:compact_array,
[
partition_index: {:int32, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, []}
]

defp request_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AssignReplicasToDirs")

defp response_schema(0),
do: [
throttle_time_ms: {:int32, %{is_nullable?: false}},
error_code: {:int16, %{is_nullable?: false}},
directories:
{{:compact_array,
[
id: {:uuid, %{is_nullable?: false}},
topics:
{{:compact_array,
[
topic_id: {:uuid, %{is_nullable?: false}},
partitions:
{{:compact_array,
[
partition_index: {:int32, %{is_nullable?: false}},
error_code: {:int16, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]}, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message AssignReplicasToDirs")
end
25 changes: 24 additions & 1 deletion lib/klife_protocol/generated/messages/broker_heartbeat.ex
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ defmodule KlifeProtocol.Messages.BrokerHeartbeat do
- current_metadata_offset: The highest metadata offset which the broker has reached. (int64 | versions 0+)
- want_fence: True if the broker wants to be fenced, false otherwise. (bool | versions 0+)
- want_shut_down: True if the broker wants to be shut down, false otherwise. (bool | versions 0+)
- offline_log_dirs: Log directories that failed and went offline. ([]uuid | versions 1+)
"""
def serialize_request(%{headers: headers, content: content}, version) do
Expand Down Expand Up @@ -83,7 +84,7 @@ defmodule KlifeProtocol.Messages.BrokerHeartbeat do
@doc """
Returns the current max supported version of this message.
"""
def max_supported_version(), do: 0
def max_supported_version(), do: 1

@doc """
Returns the current min supported version of this message.
Expand All @@ -106,6 +107,18 @@ defmodule KlifeProtocol.Messages.BrokerHeartbeat do
tag_buffer: {:tag_buffer, []}
]

defp request_schema(1),
do: [
broker_id: {:int32, %{is_nullable?: false}},
broker_epoch: {:int64, %{is_nullable?: false}},
current_metadata_offset: {:int64, %{is_nullable?: false}},
want_fence: {:boolean, %{is_nullable?: false}},
want_shut_down: {:boolean, %{is_nullable?: false}},
tag_buffer:
{:tag_buffer,
[offline_log_dirs: {{"0", {:compact_array, :uuid}}, %{is_nullable?: false}}]}
]

defp request_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message BrokerHeartbeat")

Expand All @@ -119,6 +132,16 @@ defmodule KlifeProtocol.Messages.BrokerHeartbeat do
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(1),
do: [
throttle_time_ms: {:int32, %{is_nullable?: false}},
error_code: {:int16, %{is_nullable?: false}},
is_caught_up: {:boolean, %{is_nullable?: false}},
is_fenced: {:boolean, %{is_nullable?: false}},
should_shut_down: {:boolean, %{is_nullable?: false}},
tag_buffer: {:tag_buffer, %{}}
]

defp response_schema(unkown_version),
do: raise("Unknown version #{unkown_version} for message BrokerHeartbeat")
end
Loading

0 comments on commit 6b14584

Please sign in to comment.