8000 fix(kafka and derivatives): add `alias` config to avoid clashes with same topic by thalesmg · Pull Request #13327 · emqx/emqx · GitHub
[go: up one dir, main page]
Skip to content

fix(kafka and derivatives): add alias config to avoid clashes with same topic #13327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jun 28, 2024
Merged

fix(kafka and derivatives): add alias config to avoid clashes with same topic #13327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6023012
fix(kafka and derivatives): add `alias` config to avoid clashes with …
thalesmg Jun 24, 2024
ed5e659
fix(buffer worker, kafka): send reply when async call fails immediately
thalesmg Jun 24, 2024
164a507
test(pulsar): add testcase for different producers using the same topic
thalesmg Jun 24, 2024
4c3c86e
chore: bump wolff -> 2.0.0
thalesmg Jun 26, 2024
4bd0abc
chore: bump app vsns
thalesmg Jun 27, 2024
067beec
chore: add count to thrown error
thalesmg Jun 28, 2024
fd49f66
docs: improve descriptions
thalesmg Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,14 @@ request(Method, Path, Params) ->
Error
end.

simplify_result(Res) ->
case Res of
{error, {{_, Status, _}, _, Body}} ->
{Status, Body};
{ok, {{_, Status, _}, _, Body}} ->
{Status, Body}
end.

list_bridges_api() ->
Params = [],
Path = emqx_mgmt_api_test_util:api_path(["actions"]),
Expand Down Expand Up @@ -321,7 +329,7 @@ get_bridge_api(BridgeKind, BridgeType, BridgeName) ->
Path = emqx_mgmt_api_test_util:api_path([Root, BridgeId]),
ct:pal("get bridge ~p (via http)", [{BridgeKind, BridgeType, BridgeName}]),
Res = request(get, Path, Params),
ct:pal("get bridge ~p result: ~p", [{BridgeKind, BridgeType, BridgeName}, Res]),
ct:pal("get bridge ~p result:\n ~p", [{BridgeKind, BridgeType, BridgeName}, Res]),
Res.

create_bridge_api(Config) ->
Expand Down Expand Up @@ -349,6 +357,26 @@ create_kind_api(Config, Overrides) ->
ct:pal("bridge create (~s, http) result:\n ~p", [Kind, Res]),
Res.

enable_kind_api(Kind, ConnectorType, ConnectorName) ->
do_enable_disable_kind_api(Kind, ConnectorType, ConnectorName, enable).

disable_kind_api(Kind, ConnectorType, ConnectorName) ->
do_enable_disable_kind_api(Kind, ConnectorType, ConnectorName, disable).

do_enable_disable_kind_api(Kind, Type, Name, Op) ->
BridgeId = emqx_bridge_resource:bridge_id(Type, Name),
RootBin = api_path_root(Kind),
{OpPath, OpStr} =
case Op of
enable -> {"true", "enable"};
disable -> {"false", "disable"}
end,
Path = emqx_mgmt_api_test_util:api_path([RootBin, BridgeId, "enable", OpPath]),
ct:pal(OpStr ++ " ~s ~s (http)", [Kind, BridgeId]),
Res = request(put, Path, []),
ct:pal(OpStr ++ " ~s ~s (http) result:\n ~p", [Kind, BridgeId, Res]),
simplify_result(Res).

create_connector_api(Config) ->
create_connector_api(Config, _Overrides = #{}).

Expand Down Expand Up @@ -453,6 +481,15 @@ update_bridge_api(Config, Overrides) ->
ct:pal("update bridge (~s, http) result:\n ~p", [Kind, Res]),
Res.

delete_kind_api(Kind, Type, Name) ->
BridgeId = emqx_bridge_resource:bridge_id(Type, Name),
PathRoot = api_path_root(Kind),
Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId]),
ct:pal("deleting bridge (~s, http)", [Kind]),
Res = request(delete, Path, _Params = []),
ct:pal("delete bridge (~s, http) result:\n ~p", [Kind, Res]),
simplify_result(Res).

op_bridge_api(Op, BridgeType, BridgeName) ->
op_bridge_api(_Kind = action, Op, BridgeType, BridgeName).

Expand Down
2 changes: 1 addition & 1 deletion apps/emqx_bridge_azure_event_hub/rebar.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

{erl_opts, [debug_info]}.
{deps, [
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.10.5"}}},
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}},
{kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}},
{brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}},
{brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}},
Expand Down
45 changes: 31 additions & 14 deletions apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ init_per_suite(Config) ->
emqx,
emqx_management,
emqx_resource,
%% Just for test helpers
brod,
emqx_bridge_azure_event_hub,
emqx_bridge,
emqx_rule_engine,
Expand Down Expand Up @@ -93,25 +95,23 @@ common_init_per_testcase(TestCase, Config) ->
{connector_type, ?CONNECTOR_TYPE},
{connector_name, Name},
{connector_config, ConnectorConfig},
{action_type, ?BRIDGE_TYPE},
{action_name, Name},
{action_config, BridgeConfig},
{bridge_type, ?BRIDGE_TYPE},
{bridge_name, Name},
{bridge_config, BridgeConfig}
| Config
].

end_per_testcase(_Testcase, Config) ->
case proplists:get_bool(skip_does_not_apply, Config) of
true ->
ok;
false ->
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(),
emqx_common_test_helpers:call_janitor(60_000),
ok = snabbkaffe:stop(),
ok
end.
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(),
emqx_common_test_helpers:call_janitor(60_000),
ok = snabbkaffe:stop(),
ok.

%%------------------------------------------------------------------------------
%% Helper fns
Expand Down Expand Up @@ -172,7 +172,7 @@ bridge_config(Name, ConnectorId, KafkaTopic) ->
#{
<<"enable">> => true,
<<"connector">> => ConnectorId,
<<"kafka">> =>
<<"parameters">> =>
#{
<<"buffer">> =>
#{
Expand Down Expand Up @@ -322,7 +322,7 @@ t_same_name_azure_kafka_bridges(Config) ->
),

%% then create a Kafka bridge with same name and delete it after creation
ConfigKafka0 = lists:keyreplace(bridge_type, 1, Config, {bridge_type, ?KAFKA_BRIDGE_TYPE}),
ConfigKafka0 = lists:keyreplace(action_type, 1, Config, {action_type, ?KAFKA_BRIDGE_TYPE}),
ConfigKafka = lists:keyreplace(
connector_type, 1, ConfigKafka0, {connector_type, ?KAFKA_BRIDGE_TYPE}
),
Expand Down Expand Up @@ -374,3 +374,20 @@ t_http_api_get(Config) ->
emqx_bridge_testlib:list_bridges_api()
),
ok.

t_multiple_actions_sharing_topic(Config) ->
ActionConfig0 = ?config(action_config, Config),
ActionConfig =
emqx_utils_maps:deep_merge(
ActionConfig0,
#{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}}
),
ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic(
[
{type, ?BRIDGE_TYPE_BIN},
{connector_name, ?config(connector_name, Config)},
{connector_config, ?config(connector_config, Config)},
{action_config, ActionConfig}
]
),
ok.
2 changes: 1 addition & 1 deletion apps/emqx_bridge_confluent/rebar.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

{erl_opts, [debug_info]}.
{deps, [
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.10.5"}}},
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}},
{kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}},
{brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}},
{brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}},
Expand Down
24 changes: 23 additions & 1 deletion apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ init_per_suite(Config) ->
emqx,
emqx_management,
emqx_resource,
%% Just for test helpers
brod,
emqx_bridge_confluent,
emqx_bridge,
emqx_rule_engine,
Expand Down Expand Up @@ -93,6 +95,9 @@ common_init_per_testcase(TestCase, Config) ->
{connector_type, ?CONNECTOR_TYPE},
{connector_name, Name},
{connector_config, ConnectorConfig},
{action_type, ?ACTION_TYPE},
{action_name, Name},
{action_config, BridgeConfig},
{bridge_type, ?ACTION_TYPE},
{bridge_name, Name},
{bridge_config, BridgeConfig}
Expand Down Expand Up @@ -306,7 +311,7 @@ t_same_name_confluent_kafka_bridges(Config) ->
),

%% then create a Kafka bridge with same name and delete it after creation
ConfigKafka0 = lists:keyreplace(bridge_type, 1, Config, {bridge_type, ?KAFKA_BRIDGE_TYPE}),
ConfigKafka0 = lists:keyreplace(action_type, 1, Config, {action_type, ?KAFKA_BRIDGE_TYPE}),
ConfigKafka = lists:keyreplace(
connector_type, 1, ConfigKafka0, {connector_type, ?KAFKA_BRIDGE_TYPE}
),
Expand Down Expand Up @@ -378,3 +383,20 @@ t_list_v1_bridges(Config) ->
[]
),
ok.

t_multiple_actions_sharing_topic(Config) ->
ActionConfig0 = ?config(action_config, Config),
ActionConfig =
emqx_utils_maps:deep_merge(
ActionConfig0,
#{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}}
),
ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic(
[
{type, ?ACTION_TYPE_BIN},
{connector_name, ?config(connector_name, Config)},
{connector_config, ?config(connector_config, Config)},
{action_config, ActionConfig}
]
),
ok.
2 changes: 1 addition & 1 deletion apps/emqx_bridge_kafka/rebar.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

{erl_opts, [debug_info]}.
{deps, [
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.10.5"}}},
{wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}},
{kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}},
{brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}},
{brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}},
Expand Down
2 changes: 1 addition & 1 deletion apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_kafka, [
{description, "EMQX Enterprise Kafka Bridge"},
{vsn, "0.3.2"},
{vsn, "0.3.3"},
{registered, [emqx_bridge_kafka_consumer_sup]},
{applications, [
kernel,
Expand Down
15 changes: 13 additions & 2 deletions apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand BA4F Up @@ -324,6 +324,12 @@ on_query(
}),
do_send_msg(sync, KafkaMessage, Producers, SyncTimeout)
catch
error:{invalid_partition_count, Count, _Partitioner} ->
?tp("kafka_producer_invalid_partition_count", #{
action_id => MessageTag,
query_mode => sync
}),
{error, {unrecoverable_error, {invalid_partition_count, Count}}};
throw:{bad_kafka_header, _} = Error ->
?tp(
emqx_bridge_kafka_impl_producer_sync_query_failed,
Expand Down Expand Up @@ -384,8 +390,12 @@ on_query_async(
}),
do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn)
catch
error:{invalid_partition_count, _Count, _Partitioner} ->
{error, invalid_partition_count};
error:{invalid_partition_count, Count, _Partitioner} ->
?tp("kafka_producer_invalid_partition_count", #{
action_id => MessageTag,
query_mode => async
}),
{error, {unrecoverable_error, {invalid_partition_count, Count}}};
throw:{bad_kafka_header, _} = Error ->
?tp(
emqx_bridge_kafka_impl_producer_async_query_failed,
Expand Down Expand Up @@ -690,6 +700,7 @@ producers_config(BridgeType, BridgeName, Input, IsDryRun, BridgeV2Id) ->
max_batch_bytes => MaxBatchBytes,
max_send_ahead => MaxInflight - 1,
compression => Compression,
alias => BridgeV2Id,
telemetry_meta_data => #{bridge_id => BridgeV2Id},
max_partitions => MaxPartitions
}.
Expand Down
Loading
0