8000 Prometheus integration to replicator in progress · Lispython/kafka-replicator@f821ca0 · GitHub
[go: up one dir, main page]
Skip to content

Commit f821ca0

Browse files
LispythonLispython
committed
Prometheus integration to replicator in progress
Add prometheus exporter and metrics to replicator. Add show_progress_interval_secs to route config. Add update_metrics_interval_secs to route config. Add upstream_poll_interval_ms to route config. Add optional name field to route config.
1 parent 90844e0 commit f821ca0

File tree

5 files changed

+609
-212
lines changed

5 files changed

+609
-212
lines changed

‎examples/example-config.yml

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
clusters:
22
- name: cluster_1
33
hosts:
4-
- replicator-kafka-1:9092
5-
- replicator-kafka-1:9092
4+
#- replicator-kafka-1:9092
5+
#- replicator-kafka-1:9092
66
- kafka_replicator_replicator_kafka_1_1:9092
77
- name: cluster_2
88
hosts:
99
- kafka_replicator_replicator_kafka_2_1:9092
10-
- replicator-kafka-2:9092
10+
#- replicator-kafka-2:9092
1111

1212
clients:
1313
- client: cl_1_client_1
@@ -19,21 +19,21 @@ clients:
1919
cluster: cluster_2
2020

2121
routes:
22-
- upstream_client: cl_1_client_1
23-
downstream_client: cl_1_client_1
24-
upstream_topics:
22+
- upstream_client: cl_1_client_1 # Required: source client name from clients section
23+
downstream_client: cl_1_client_1 # Required: target client name from clients section
24+
upstream_topics: # Required: source topics
2525
- 'topic1'
26-
downstream_topic: 'topic2'
26+
downstream_topic: 'topic2' # Required: target topics
2727 10000
repartitioning_strategy: random # strict_p2p | random
28-
upstream_group_id: group_22
29-
show_progress_interval_secs: 10
28+
upstream_group_id: group_22 # Option: upstream consumer group id
29+
show_progress_interval_secs: 2 # Optional: interval between console output
30+
update_metrics_interval_secs: 1 # Optional: interval between prometheus metrics updates
31+
upstream_poll_interval_ms: 200 # Optional: interval between upstream poll calls
32+
name: route_name # Optional:
3033
limits:
3134
messages_per_sec: 10000
3235
number_of_messages:
3336

34-
labels:
35-
label_key: label_value
36-
3737
- upstream_client: cl_1_client_1
3838
downstream_client: cl_2_client_1
3939
upstream_topics:
@@ -60,29 +60,24 @@ observers:
6060
group_id: "group_id"
6161
topics:
6262
- 'topic1'
63-
- 'topic2'
63+
# - 'topic2'
6464
fetch_timeout_secs: 5
6565
fetch_interval_secs: 5
66-
show_progress_interval_secs: 5
67-
labels:
68-
label_key: label_value
69-
env: prod
66+
show_progress_interval_secs: 1
67+
update_metrics_interval_secs: 10
7068

7169
- client: cl_2_client_1
7270
topic: 'topic3'
7371
topics:
7472
- 'topic2'
7573
show_progress_interval_secsy: 20
76-
labels:
77-
label_key: label_value2
78-
env: stage
7974

8075
- client: cl_1_client_1
8176
topic: 'topic1'
8277
topics: []
8378

8479
prometheus:
85-
namespace: "app:observer:"
80+
# namespace: "app:observer:"
8681
labels:
8782
label_key: label_value
8883
env: prod

0 commit comments

Comments
 (0)
0