8000 Add workqueue prometheus metrics · nginx/kubernetes-ingress@36d3323 · GitHub
[go: up one dir, main page]
Skip to content

Commit 36d3323

Browse files
Dean-CoakleyDean-Coakley
authored
Add workqueue prometheus metrics
* nginx_ingress_controller_workqueue_depth * nginx_ingress_controller_workqueue_queue_duration_seconds * nginx_ingress_controller_workqueue_work_duration_seconds
1 parent 3d6a414 commit 36d3323

File tree

5 files changed

+136
-21
lines changed

5 files changed

+136
-21
lines changed

‎cmd/nginx-ingress/main.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,7 @@ func main() {
368368
managerCollector = collectors.NewLocalManagerMetricsCollector(constLabels)
369369
controllerCollector = collectors.NewControllerMetricsCollector(*enableCustomResources, constLabels)
370370
processCollector := collectors.NewNginxProcessesMetricsCollector(constLabels)
371+
workQueueCollector := collectors.NewWorkQueueMetricsCollector(constLabels)
371372

372373
err = managerCollector.Register(registry)
373374
if err != nil {
@@ -383,6 +384,11 @@ func main() {
383384
if err != nil {
384385
glog.Errorf("Error registering NginxProcess Prometheus metrics: %v", err)
385386
}
387+
388+
err = workQueueCollector.Register(registry)
389+
if err != nil {
390+
glog.Errorf("Error registering WorkQueue Prometheus metrics: %v", err)
391+
}
386392
}
387393

388394
useFakeNginxManager := *proxyURL != ""

‎docs-web/logging-and-monitoring/prometheus.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ The Ingress Controller exports the following metrics:
2727
* Exported by NGINX/NGINX Plus. Refer to the [NGINX Prometheus Exporter developer docs](https://github.com/nginxinc/nginx-prometheus-exporter#exported-metrics) to find more information about the exported metrics.
2828
* There is a Grafana dashboard for NGINX Plus metrics located in the root repo folder.
2929
* Calculated by the Ingress Controller:
30-
* `controller_upstream_server_response_latency_ms_count`. Bucketed response times from when NGINX establishes a connection to an upstream server to when the last byte of the response body is received by NGINX. **Note**: The metric for the upstream isn't available until traffic is sent to the upstream. The metric isn't enabled by default. To enable the metric, set the `-enable-latency-metrics` command-line argument.
30+
* `controller_upstream_server_response_latency_ms_count`. Bucketed response times from when NGINX establishes a connection to an upstream server to when the last byte of the response body is received by NGINX. **Note**: The metric for the upstream isn't available until traffic is sent to the upstream. The metric isn't enabled by default. To enable the metric, set the `-enable-latency-metrics` command-line argument.
3131
* Ingress Controller metrics
3232
* `controller_nginx_reloads_total`. Number of successful NGINX reloads. This includes the label `reason` with 2 possible values `endpoints` (the reason for the reload was an endpoints update) and `other` (the reload was caused by something other than an endpoint update like an ingress update).
3333
* `controller_nginx_reload_errors_total`. Number of unsuccessful NGINX reloads.
@@ -37,7 +37,11 @@ The Ingress Controller exports the following metrics:
3737
* `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration)). **Note**: The metric doesn't count minions without a master.
3838
* `controller_virtualserver_resources_total`. Number of handled VirtualServer resources.
3939
* `controller_virtualserverroute_resources_total`. Number of handled VirtualServerRoute resources. **Note**: The metric counts only VirtualServerRoutes that have a reference from a VirtualServer.
40+
* Workqueue metrics. **Note**: the workqueue is a queue used by the Ingress Controller to process changes to the relevant resources in the cluster like Ingress resources. The Ingress Controller uses only one queue. The metrics for that queue will have the label `name="taskQueue"`
41+
* `workqueue_depth`. Current depth of the workqueue.
42+
* `workqueue_queue_duration_second`. How long in seconds an item stays in the workqueue before being requested.
43+
* `workqueue_work_duration_seconds`. How long in seconds processing an item from the workqueue takes.
4044

41-
**Note**: all metrics have the namespace nginx_ingress. For example, nginx_ingress_controller_nginx_reloads_total.
45+
**Note**: all metrics have the namespace `nginx_ingress`. For example, `nginx_ingress_controller_nginx_reloads_total`.
4246

4347
**Note**: all metrics include the label `class`, which is set to the class of the Ingress Controller. The class is configured via the `-ingress-class` command-line argument.

‎internal/k8s/task_queue.go‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ type taskQueue struct {
2929
// The sync function is called for every element inserted into the queue.
3030
func newTaskQueue(syncFn func(task)) *taskQueue {
3131
return &taskQueue{
32-
queue: workqueue.New(),
32+
queue: workqueue.NewNamed("taskQueue"),
3333
sync: syncFn,
3434
workerDone: make(chan struct{}),
3535
}
@@ -55,7 +55,6 @@ func (tq *taskQueue) Enqueue(obj interface{}) {
5555
}
5656

5757
glog.V(3).Infof("Adding an element with a key: %v", task.Key)
58-
5958
tq.queue.Add(task)
6059
}
6160

@@ -103,30 +102,19 @@ func (tq *taskQueue) Shutdown() {
103102
// kind represents the kind of the Kubernetes resources of a task
104103
type kind int
105104

105+
// resources
106106
const (
107-
// ingress resource
108107
ingress = iota
109-
// endpoints resource
110108
endpoints
111-
// configMap resource
112109
configMap
113-
// secret resource
114110
secret
115-
// service resource
116111
service
117-
// virtualserver resource
118112
virtualserver
119-
// virtualServeRoute resource
120113
virtualServerRoute
121-
// globalConfiguration resource
122114
globalConfiguration
123-
// transportserver resource
124115
transportserver
125-
// policy resource
126116
policy
127-
// appProtectPolicy resource
128117
appProtectPolicy
129-
// appProtectlogconf resource
130118
appProtectLogConf
131119
)
132120

@@ -166,7 +154,7 @@ func newTask(key string, obj interface{}) (task, error) {
166154
} else if objectKind == appProtectLogConfGVK.Kind {
167155
k = appProtectLogConf
168156
} else {
169-
return task{}, fmt.Errorf("Unknow unstructured kind: %v", objectKind)
157+
return task{}, fmt.Errorf("Unknown unstructured kind: %v", objectKind)
170158
}
171159
default:
172160
return task{}, fmt.Errorf("Unknown type: %v", t)

‎internal/metrics/collectors/processes.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,14 @@ import (
1010
"github.com/prometheus/client_golang/prometheus"
1111
)
1212

13-
// NginxProcessesMetricsCollector implements NginxPorcessesCollector interface and prometheus.Collector interface
13+
// NginxProcessesMetricsCollector implements prometheus.Collector interface
1414
type NginxProcessesMetricsCollector struct {
15-
// Metrics
1615
workerProcessTotal *prometheus.GaugeVec
1716
}
1817

1918
// NewNginxProcessesMetricsCollector creates a new NginxProcessMetricsCollector
2019
func NewNginxProcessesMetricsCollector(constLabels map[string]string) *NginxProcessesMetricsCollector {
21-
pc := &NginxProcessesMetricsCollector{
20+
return &NginxProcessesMetricsCollector{
2221
workerProcessTotal: prometheus.NewGaugeVec(
2322
prometheus.GaugeOpts{
2423
Name: "nginx_worker_processes_total",
@@ -29,7 +28,6 @@ func NewNginxProcessesMetricsCollector(constLabels map[string]string) *NginxProc
2928
[]string{"generation"},
3029
),
3130
}
32-
return pc
3331
}
3432

3533
// updateWorkerProcessCount sets the number of NGINX worker processes

‎internal/metrics/collectors/workqueue.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package collectors
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"k8s.io/client-go/util/workqueue"
6+
)
7+
8+
// WorkQueueMetricsCollector collects the metrics about the work queue, which the Ingress Controller uses to process changes to the resources in the cluster.
9+
// implements the prometheus.Collector interface
10+
type WorkQueueMetricsCollector struct {
11+
depth *prometheus.GaugeVec
12+
latency *prometheus.HistogramVec
13+
workDuration *prometheus.HistogramVec
14+
}
15+
16+
// NewWorkQueueMetricsCollector creates a new WorkQueueMetricsCollector
17+
func NewWorkQueueMetricsCollector(constLabels map[string]string) *WorkQueueMetricsCollector {
18+
const workqueueSubsystem = "workqueue"
19+
var latencyBucketSeconds = []float64{0.1, 0.5, 1, 5, 10, 50}
20+
21+
return &WorkQueueMetricsCollector{
22+
depth: prometheus.NewGaugeVec(
23+
prometheus.GaugeOpts{
24+
Namespace: metricsNamespace,
25+
Subsystem: workqueueSubsystem,
26+
Name: "depth",
27+
Help: "Current depth of workqueue",
28+
ConstLabels: constLabels,
29+
},
30+
[]string{"name"},
31+
),
32+
latency: prometheus.NewHistogramVec(
33+
prometheus.HistogramOpts{
34+
Namespace: metricsNamespace,
35+
Subsystem: workqueueSubsystem,
36+
Name: "queue_duration_seconds",
37+
Help: "How long in seconds an item stays in workqueue before being processed",
38+
Buckets: latencyBucketSeconds,
39+
ConstLabels: constLabels,
40+
},
41+
[]string{"name"},
42+
),
43+
workDuration: prometheus.NewHistogramVec(
44+
prometheus.HistogramOpts{
45+
Namespace: metricsNamespace,
46+
Subsystem: workqueueSubsystem,
47+
Name: "work_duration_seconds",
48+
Help: "How long in seconds processing an item from workqueue takes",
49+
Buckets: latencyBucketSeconds,
50+
ConstLabels: constLabels,
51+
},
52+
[]string{"name"},
53+
),
54+
}
55+
}
56+
57+
// Collect implements the prometheus.Collector interface Collect method
58+
func (wqc *WorkQueueMetricsCollector) Collect(ch chan<- prometheus.Metric) {
59+
wqc.depth.Collect(ch)
60+
wqc.latency.Collect(ch)
61+
wqc.workDuration.Collect(ch)
62+
}
63+
64+
// Describe implements the prometheus.Collector interface Describe method
65+
func (wqc *WorkQueueMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
66+
wqc.depth.Describe(ch)
67+
wqc.latency.Describe(ch)
68+
wqc.workDuration.Describe(ch)
69+
}
70+
71+
// Register registers all the metrics of the collector
72+
func (wqc *WorkQueueMetricsCollector) Register(registry *prometheus.Registry) error {
73+
workqueue.SetProvider(wqc)
74+
return registry.Register(wqc)
75+
}
76+
77+
// NewDepthMetric implements the workqueue.MetricsProvider interface NewDepthMetric method
78+
func (wqc *WorkQueueMetricsCollector) NewDepthMetric(name string) workqueue.GaugeMetric {
79+
return wqc.depth.WithLabelValues(name)
80+
}
81+
82+
// NewLatencyMetric implements the workqueue.MetricsProvider interface NewLatencyMetric method
83+
func (wqc *WorkQueueMetricsCollector) NewLatencyMetric(name string) workqueue.HistogramMetric {
84+
return wqc.latency.WithLabelValues(name)
85+
86+
}
87+
88+
// NewWorkDurationMetric implements the workqueue.MetricsProvider interface NewWorkDurationMetric method
89+
func (wqc *WorkQueueMetricsCollector) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
90+
return wqc.workDuration.WithLabelValues(name)
91+
}
92+
93+
// noopMetric implements the workqueue.GaugeMetric and workqueue.HistogramMetric interfaces
94+
type noopMetric struct{}
95+
96+
func (noopMetric) Inc() {}
97+
func (noopMetric) Dec() {}
98+
func (noopMetric) Set(float64) {}
99+
func (noopMetric) Observe(float64) {}
100+
101+
// NewAddsMetric implements the workqueue.MetricsProvider interface NewAddsMetric method
102+
func (*WorkQueueMetricsCollector) NewAddsMetric(string) workqueue.CounterMetric {
103+
return noopMetric{}
104+
}
105+
106+
// NewUnfinishedWorkSecondsMetric implements the workqueue.MetricsProvider interface NewUnfinishedWorkSecondsMetric method
107+
func (*WorkQueueMetricsCollector) NewUnfinishedWorkSecondsMetric(string) workqueue.SettableGaugeMetric {
108+
return noopMetric{}
109+
}
110+
111+
// NewLongestRunningProcessorSecondsMetric implements the workqueue.MetricsProvider interface NewLongestRunningProcessorSecondsMetric method
112+
func (*WorkQueueMetricsCollector) NewLongestRunningProcessorSecondsMetric(string) workqueue.SettableGaugeMetric {
113+
return noopMetric{}
114+
}
115+
116+
// NewRetriesMetric implements the workqueue.MetricsProvider interface NewRetriesMetric method
117+
func (*WorkQueueMetricsCollector) NewRetriesMetric(string) workqueue.CounterMetric {
118+
return noopMetric{}
119+
}

0 commit comments

Comments
 (0)
0