8000 Add workqueue prometheus metrics · nginx/kubernetes-ingress@6984981 · GitHub
[go: up one dir, main page]
Skip to content

Commit 6984981

Browse files
Dean-CoakleyDean-Coakley
committed
Add workqueue prometheus metrics
1 parent 4285b29 commit 6984981

File tree

5 files changed

+133
-17
lines changed

5 files changed

+133
-17
lines changed

‎cmd/nginx-ingress/main.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,7 @@ func main() {
367367
managerCollector = collectors.NewLocalManagerMetricsCollector(constLabels)
368368
controllerCollector = collectors.NewControllerMetricsCollector(*enableCustomResources, constLabels)
369369
processCollector := collectors.NewNginxProcessesMetricsCollector(constLabels)
370+
workQueueCollector := collectors.NewWorkQueueMetricsCollector(constLabels)
370371

371372
err = managerCollector.Register(registry)
372373
if err != nil {
@@ -382,6 +383,11 @@ func main() {
382383
if err != nil {
383384
glog.Errorf("Error registering NginxProcess Prometheus metrics: %v", err)
384385
}
386+
387+
err = workQueueCollector.Register(registry)
388+
if err != nil {
389+
glog.Errorf("Error registering WorkQueue Prometheus metrics: %v", err)
390+
}
385391
}
386392

387393
useFakeNginxManager := *proxyURL != ""

‎docs-web/logging-and-monitoring/prometheus.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ The Ingress Controller exports the following metrics:
2727
* Exported by NGINX/NGINX Plus. Refer to the [NGINX Prometheus Exporter developer docs](https://github.com/nginxinc/nginx-prometheus-exporter#exported-metrics) to find more information about the exported metrics.
2828
* There is a Grafana dashboard for NGINX Plus metrics located in the root repo folder.
2929
* Calculated by the Ingress Controller:
30-
* `controller_upstream_server_response_latency_ms_count`. Bucketed response times from when NGINX establishes a connection to an upstream server to when the last byte of the response body is received by NGINX. **Note**: The metric for the upstream isn't available until traffic is sent to the upstream. The metric isn't enabled by default. To enable the metric, set the `-enable-latency-metrics` command-line argument.
30+
* `controller_upstream_server_response_latency_ms_count`. Bucketed response times from when NGINX establishes a connection to an upstream server to when the last byte of the response body is received by NGINX. **Note**: The metric for the upstream isn't available until traffic is sent to the upstream. The metric isn't enabled by default. To enable the metric, set the `-enable-latency-metrics` command-line argument.
3131
* Ingress Controller metrics
3232
* `controller_nginx_reloads_total`. Number of successful NGINX reloads. This includes the label `reason` with 2 possible values `endpoints` (the reason for the reload was an endpoints update) and `other` (the reload was caused by something other than an endpoint update like an ingress update).
3333
* `controller_nginx_reload_errors_total`. Number of unsuccessful NGINX reloads.
@@ -37,6 +37,11 @@ The Ingress Controller exports the following metrics:
3737
* `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration)). **Note**: The metric doesn't count minions without a master.
3838
* `controller_virtualserver_resources_total`. Number of handled VirtualServer resources.
3939
* `controller_virtualserverroute_resources_total`. Number of handled VirtualServerRoute resources. **Note**: The metric counts only VirtualServerRoutes that have a reference from a VirtualServer.
40+
* Kubernetes Cluster metrics
41+
* `controller_workqueue_depth` Current depth of workqueue.
42+
* `controller_workqueue_queue_duration_second`. How long in seconds an item stays in workqueue before being requested.
43+
* `controller_workqueue_work_duration_seconds`. How long in seconds processing an item from workqueue takes.
44+
4045

4146
**Note**: all metrics have the namespace nginx_ingress. For example, nginx_ingress_controller_nginx_reloads_total.
4247

‎internal/k8s/task_queue.go‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ type taskQueue struct {
2929
// The sync function is called for every element inserted into the queue.
3030
func newTaskQueue(syncFn func(task)) *taskQueue {
3131
return &taskQueue{
32-
queue: workqueue.New(),
32+
queue: workqueue.NewNamed("taskQueue"),
3333
sync: syncFn,
3434
workerDone: make(chan struct{}),
3535
}
@@ -55,7 +55,6 @@ func (tq *taskQueue) Enqueue(obj interface{}) {
5555
}
5656

5757
glog.V(3).Infof("Adding an element with a key: %v", task.Key)
58-
5958
tq.queue.Add(task)
6059
}
6160

@@ -103,30 +102,19 @@ func (tq *taskQueue) Shutdown() {
103102
// kind represents the kind of the Kubernetes resources of a task
104103
type kind int
105104

105+
// resources
106106
const (
107-
// ingress resource
108107
ingress = iota
109-
// endpoints resource
110108
endpoints
111-
// configMap resource
112109
configMap
113-
// secret resource
114110
secret
115-
// service resource
116111
service
117-
// virtualserver resource
118112
virtualserver
119-
// virtualServeRoute resource
120113
virtualServerRoute
121-
// globalConfiguration resource
122114
globalConfiguration
123-
// transportserver resource
124115
transportserver
125-
// policy resource
126116
policy
127-
// appProtectPolicy resource
128117
appProtectPolicy
129-
// appProtectlogconf resource
130118
appProtectLogConf
131119
)
132120

@@ -166,7 +154,7 @@ func newTask(key string, obj interface{}) (task, error) {
166154
} else if objectKind == appProtectLogConfGVK.Kind {
167155
k = appProtectLogConf
168156
} else {
169-
return task{}, fmt.Errorf("Unknow unstructured kind: %v", objectKind)
157+
return task{}, fmt.Errorf("Unknown unstructured kind: %v", objectKind)
170158
}
171159
default:
172160
return task{}, fmt.Errorf("Unknown type: %v", t)

‎internal/metrics/collectors/processes.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/prometheus/client_golang/prometheus"
1111
)
1212

13-
// NginxProcessesMetricsCollector implements NginxPorcessesCollector interface and prometheus.Collector interface
13+
// NginxProcessesMetricsCollector implements NginxProcessesCollector interface and prometheus.Collector interface
1414
type NginxProcessesMetricsCollector struct {
1515
// Metrics
1616
workerProcessTotal *prometheus.GaugeVec

‎internal/metrics/collectors/workqueue.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package collectors
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"k8s.io/client-go/util/workqueue"
6+
)
7+
8+
const workqueueSubsystem = "workqueue"
9+
10+
// WorkQueueMetricsCollector implements prometheus.Collector interface
11+
type WorkQueueMetricsCollector struct {
12+
depth *prometheus.GaugeVec
13+
latency *prometheus.HistogramVec
14+
workDuration *prometheus.HistogramVec
15+
}
16+
17+
// NewWorkQueueMetricsCollector creates a new WorkQueueMetricsCollec E7ED tor
18+
func NewWorkQueueMetricsCollector(constLabels map[string]string) *WorkQueueMetricsCollector {
19+
return &WorkQueueMetricsCollector{
20+
depth: prometheus.NewGaugeVec(
21+
prometheus.GaugeOpts{
22+
Namespace: metricsNamespace,
23+
Subsystem: workqueueSubsystem,
24+
Name: "depth",
25+
Help: "Current depth of workqueue",
26+
ConstLabels: constLabels,
27+
},
28+
[]string{"name"},
29+
),
30+
latency: prometheus.NewHistogramVec(
31+
prometheus.HistogramOpts{
32+
Namespace: metricsNamespace,
33+
Subsystem: workqueueSubsystem,
34+
Name: "queue_duration_seconds",
35+
Help: "How long in seconds an item stays in workqueue before being requested",
36+
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
37+
ConstLabels: constLabels,
38+
},
39+
[]string{"name"},
40+
),
41+
workDuration: prometheus.NewHistogramVec(
42+
prometheus.HistogramOpts{
43+
Namespace: metricsNamespace,
44+
Subsystem: workqueueSubsystem,
45+
Name: "work_duration_seconds",
46+
Help: "How long in seconds processing an item from workqueue takes",
47+
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
48+
ConstLabels: constLabels,
49+
},
50+
[]string{"name"},
51+
),
52+
}
53+
}
54+
55+
// Collect implements the prometheus.Collector interface Collect method
56+
func (wqc *WorkQueueMetricsCollector) Collect(ch chan<- prometheus.Metric) {
57+
wqc.depth.Collect(ch)
58+
wqc.latency.Collect(ch)
59+
wqc.workDuration.Collect(ch)
60+
}
61+
62+
// Describe implements prometheus.Collector interface Describe method
63+
func (wqc *WorkQueueMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
64+
wqc.depth.Describe(ch)
65+
wqc.latency.Describe(ch)
66+
wqc.workDuration.Describe(ch)
67+
}
68+
69+
// Register registers all the metrics of the collector
70+
func (wqc *WorkQueueMetricsCollector) Register(registry *prometheus.Registry) error {
71+
workqueue.SetProvider(wqc)
72+
return registry.Register(wqc)
73+
}
74+
75+
// NewDepthMetric implements the workqueue.MetricsProvider interface NewDepthMetric method
76+
func (wqc *WorkQueueMetricsCollector) NewDepthMetric(name string) workqueue.GaugeMetric {
77+
return wqc.depth.WithLabelValues(name)
78+
}
79+
80+
// NewLatencyMetric implements the workqueue.MetricsProvider interface NewLatencyMetric method
81+
func (wqc *WorkQueueMetricsCollector) NewLatencyMetric(name string) workqueue.HistogramMetric {
82+
return wqc.latency.WithLabelValues(name)
83+
84+
}
85+
86+
// NewWorkDurationMetric implements the workqueue.MetricsProvider interface NewWorkDurationMetric method
87+
func (wqc *WorkQueueMetricsCollector) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
88+
return wqc.workDuration.WithLabelValues(name)
89+
}
90+
91+
// noopMetric implements the workqueue.GaugeMetric and workqueue.HistogramMetric interfaces
92+
type noopMetric struct{}
93+
94+
func (noopMetric) Inc() {}
95+
func (noopMetric) Dec() {}
96+
func (noopMetric) Set(float64) {}
97+
func (noopMetric) Observe(float64) {}
98+
99+
// NewAddsMetric implements the workqueue.MetricsProvider interface NewAddsMetric method
100+
func (*WorkQueueMetricsCollector) NewAddsMetric(string) workqueue.CounterMetric {
101+
return noopMetric{}
102+
}
103+
104+
// NewUnfinishedWorkSecondsMetric implements the workqueue.MetricsProvider interface NewUnfinishedWorkSecondsMetric method
105+
func (*WorkQueueMetricsCollector) NewUnfinishedWorkSecondsMetric(string) workqueue.SettableGaugeMetric {
106+
return noopMetric{}
107+
}
108+
109+
// NewLongestRunningProcessorSecondsMetric implements the workqueue.MetricsProvider interface NewLongestRunningProcessorSecondsMetric method
110+
func (*WorkQueueMetricsCollector) NewLongestRunningProcessorSecondsMetric(string) workqueue.SettableGaugeMetric {
111+
return noopMetric{}
112+
}
113+
114+
// NewRetriesMetric implements the workqueue.MetricsProvider interface NewRetriesMetric method
115+
func (*WorkQueueMetricsCollector) NewRetriesMetric(string) workqueue.CounterMetric {
116+
return noopMetric{}
117+
}

0 commit comments

Comments
 (0)
0