8000 Add metric about total number of TransportServers by pleshakov · Pull Request #2149 · nginx/kubernetes-ingress · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/content/logging-and-monitoring/prometheus.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ The Ingress Controller exports the following metrics:
* `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration)). **Note**: The metric doesn't count minions without a master.
* `controller_virtualserver_resources_total`. Number of handled VirtualServer resources.
* `controller_virtualserverroute_resources_total`. Number of handled VirtualServerRoute resources. **Note**: The metric counts only VirtualServerRoutes that have a reference from a VirtualServer.
* `controller_transportserver_resources_total`. Number of handled TransportServer resources. This metric includes the label type, that groups the TransportServer resources by their type (passthrough, tcp or udp).
* Workqueue metrics. **Note**: the workqueue is a queue used by the Ingress Controller to process changes to the relevant resources in the cluster like Ingress resources. The Ingress Controller uses only one queue. The metrics for that queue will have the label `name="taskQueue"`
* `workqueue_depth`. Current depth of the workqueue.
* `workqueue_queue_duration_second`. How long in seconds an item stays in the workqueue before being requested.
Expand Down
31 changes: 31 additions & 0 deletions internal/k8s/configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,13 @@ func compareObjectMetasWithAnnotations(meta1 *metav1.ObjectMeta, meta2 *metav1.O
return compareObjectMetas(meta1, meta2) && reflect.DeepEqual(meta1.Annotations, meta2.Annotations)
}

// TransportServerMetrics holds metrics about TransportServer resources
type TransportServerMetrics struct {
TotalTLSPassthrough int
TotalTCP int
TotalUDP int
}

// Configuration represents the configuration of the Ingress Controller - a collection of configuration objects
// (Ingresses, VirtualServers, VirtualServerRoutes) ready to be transformed into NGINX config.
// It holds the latest valid state of those objects.
Expand Down Expand Up @@ -1442,6 +1449,30 @@ func (c *Configuration) buildVirtualServerRoutes(vs *conf_v1.VirtualServer) ([]*
return vsrs, warnings
}

// GetTransportServerMetrics returns metrics about TransportServers
func (c *Configuration) GetTransportServerMetrics() *TransportServerMetrics {
var metrics TransportServerMetrics

if c.isTLSPassthroughEnabled {
for _, resource := range c.hosts {
_, ok := resource.(*TransportServerConfiguration)
if ok {
metrics.TotalTLSPassthrough++
}
}
}

for _, tsConfig := range c.listeners {
if tsConfig.TransportServer.Spec.Listener.Protocol == "TCP" {
metrics.TotalTCP++
} else {
metrics.TotalUDP++
}
}

return &metrics
}

func getSortedIngressKeys(m map[string]*networking.Ingress) []string {
var keys []string

Expand Down
98 changes: 98 additions & 0 deletions internal/k8s/configuration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3256,6 +3256,104 @@ func TestGetResources(t *testing.T) {
}
}

func TestGetTransportServerMetrics(t *testing.T) {
tsPass := createTestTLSPassthroughTransportServer("transportserver", "abc.example.com")
tsTCP := createTestTransportServer("transportserver-tcp", "tcp-7777", "TCP")
tsUDP := createTestTransportServer("transportserver-udp", "udp-7777", "UDP")

tests := []struct {
tses []*conf_v1alpha1.TransportServer
expected *TransportServerMetrics
msg string
}{
{
tses: nil,
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 0,
TotalUDP: 0,
},
msg: "no TransportServers",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsPass,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 1,
TotalTCP: 0,
TotalUDP: 0,
},
msg: "one TLSPassthrough TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsTCP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 1,
TotalUDP: 0,
},
msg: "one TCP TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsUDP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 0,
TotalUDP: 1,
},
msg: "one UDP TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsPass, tsTCP, tsUDP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 1,
TotalTCP: 1,
TotalUDP: 1,
},
msg: "TLSPasstrough, TCP and UDP TransportServers",
},
}

listeners := []conf_v1alpha1.Listener{
{
Name: "tcp-7777",
Port: 7777,
Protocol: "TCP",
},
{
Name: "udp-7777",
Port: 7777,
Protocol: "UDP",
},
}
gc := createTestGlobalConfiguration(listeners)

for _, test := range tests {
configuration := createTestConfiguration()

_, _, err := configuration.AddOrUpdateGlobalConfiguration(gc)
if err != nil {
t.Fatalf("AddOrUpdateGlobalConfiguration() returned unexpected error %v", err)
}

for _, ts := range test.tses {
configuration.AddOrUpdateTransportServer(ts)
}

result := configuration.GetTransportServerMetrics()
if diff := cmp.Diff(test.expected, result); diff != "" {
t.Errorf("GetTransportServerMetrics() returned unexpected result for the case of %s (-want +got):\n%s", test.msg, diff)
}
}
}

func TestIsEqualForIngressConfigurationes(t *testing.T) {
regularIng := createTestIngress("regular-ingress", "foo.example.com")

Expand Down
14 changes: 13 additions & 1 deletion internal/k8s/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc

lbc.secretStore = secrets.NewLocalSecretStore(lbc.configurator)

lbc.updateIngressMetrics()
return lbc
}

Expand Down Expand Up @@ -712,6 +711,7 @@ func (lbc *LoadBalancerController) sync(task task) {
case ingress:
lbc.syncIngress(task)
lbc.update 9E12 IngressMetrics()
lbc.updateTransportServerMetrics()
case configMap:
lbc.syncConfigMap(task)
case endpoints:
Expand All @@ -723,13 +723,16 @@ func (lbc *LoadBalancerController) sync(task task) {
case virtualserver:
lbc.syncVirtualServer(task)
lbc.updateVirtualServerMetrics()
lbc.updateTransportServerMetrics()
case virtualServerRoute:
lbc.syncVirtualServerRoute(task)
lbc.updateVirtualServerMetrics()
case globalConfiguration:
lbc.syncGlobalConfiguration(task)
lbc.updateTransportServerMetrics()
case transportserver:
lbc.syncTransportServer(task)
lbc.updateTransportServerMetrics()
case policy:
lbc.syncPolicy(task)
case appProtectPolicy:
Expand Down Expand Up @@ -1638,6 +1641,15 @@ func (lbc *LoadBalancerController) updateVirtualServerMetrics() {
lbc.metricsCollector.SetVirtualServerRoutes(vsrCount)
}

func (lbc *LoadBalancerController) updateTransportServerMetrics() {
if !lbc.areCustomResourcesEnabled {
return
}

metrics := lbc.configuration.GetTransportServerMetrics()
lbc.metricsCollector.SetTransportServers(metrics.TotalTLSPassthrough, metrics.TotalTCP, metrics.TotalUDP)
}

func (lbc *LoadBalancerController) syncService(task task) {
key := task.Key
glog.V(3).Infof("Syncing service %v", key)
Expand Down
82 changes: 60 additions & 22 deletions internal/metrics/collectors/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ type ControllerCollector interface {
SetIngresses(ingressType string, count int)
SetVirtualServers(count int)
SetVirtualServerRoutes(count int)
SetTransportServers(tlsPassthroughCount, tcpCount, udpCount int)
Register(registry *prometheus.Registry) error
}

Expand All @@ -18,6 +19,7 @@ type ControllerMetricsCollector struct {
ingressesTotal *prometheus.GaugeVec
virtualServersTotal prometheus.Gauge
virtualServerRoutesTotal prometheus.Gauge
transportServersTotal *prometheus.GaugeVec
}

// NewControllerMetricsCollector creates a new ControllerMetricsCollector
Expand All @@ -32,34 +34,58 @@ func NewControllerMetricsCollector(crdsEnabled bool, constLabels map[string]stri
labelNamesController,
)

if !crdsEnabled {
return &ControllerMetricsCollector{ingressesTotal: ingResTotal}
var vsResTotal, vsrResTotal prometheus.Gauge
var tsResTotal *prometheus.GaugeVec

if crdsEnabled {
vsResTotal = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServer resources",
ConstLabels: constLabels,
},
)

vsrResTotal = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserverroute_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServerRoute resources",
ConstLabels: constLabels,
},
)

tsResTotal = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "transportserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled TransportServer resources",
ConstLabels: constLabels,
},
labelNamesController,
)
}

vsResTotal := prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServer resources",
ConstLabels: constLabels,
},
)

vsrResTotal := prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserverroute_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServerRoute resources",
ConstLabels: constLabels,
},
)

return &ControllerMetricsCollector{
crdsEnabled: true,
c := &ControllerMetricsCollector{
crdsEnabled: crdsEnabled,
ingressesTotal: ingResTotal,
virtualServersTotal: vsResTotal,
virtualServerRoutesTotal: vsrResTotal,
transportServersTotal: tsResTotal,
}

// if we don't set to 0 metrics with the label type, the metrics will not be created initially

c.SetIngresses("regular", 0)
c.SetIngresses("master", 0)
c.SetIngresses("minion", 0)

if crdsEnabled {
c.SetTransportServers(0, 0, 0)
}

return c
}

// SetIngresses sets the value of the ingress resources gauge for a given type
Expand All @@ -77,12 +103,20 @@ func (cc *ControllerMetricsCollector) SetVirtualServerRoutes(count int) {
cc.virtualServerRoutesTotal.Set(float64(count))
}

// SetTransportServers sets the value of the TransportServer resources gauge
func (cc *ControllerMetricsCollector) SetTransportServers(tlsPassthroughCount, tcpCount, udpCount int) {
cc.transportServersTotal.WithLabelValues("passthrough").Set(float64(tlsPassthroughCount))
cc.transportServersTotal.WithLabelValues("tcp").Set(float64(tcpCount))
cc.transportServersTotal.WithLabelValues("udp").Set(float64(udpCount))
}

// Describe implements prometheus.Collector interface Describe method
func (cc *ControllerMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
cc.ingressesTotal.Describe(ch)
if cc.crdsEnabled {
cc.virtualServersTotal.Describe(ch)
cc.virtualServerRoutesTotal.Describe(ch)
cc.transportServersTotal.Describe(ch)
}
}

Expand All @@ -92,6 +126,7 @@ func (cc *ControllerMetricsCollector) Collect(ch chan<- prometheus.Metric) {
if cc.crdsEnabled {
cc.virtualServersTotal.Collect(ch)
cc.virtualServerRoutesTotal.Collect(ch)
cc.transportServersTotal.Collect(ch)
}
}

Expand Down Expand Up @@ -119,3 +154,6 @@ func (cc *ControllerFakeCollector) SetVirtualServers(count int) {}

// SetVirtualServerRoutes implements a fake SetVirtualServerRoutes
func (cc *ControllerFakeCollector) SetVirtualServerRoutes(count int) {}

// SetTransportServers implements a fake SetTransportServers
func (cc *ControllerFakeCollector) SetTransportServers(int, int, int) {}
12 changes: 12 additions & 0 deletions tests/data/prometheus/transport-server/global-configuration.yaml
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: k8s.nginx.org/v1alpha1
kind: GlobalConfiguration
metadata:
name: nginx-configuration
spec:
listeners:
- name: tcp-server
port: 3333
protocol: TCP
- name: udp-server
port: 3333
protocol: UDP
15 changes: 15 additions & 0 deletions tests/data/prometheus/transport-server/passthrough.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: k8s.nginx.org/v1alpha1
kind: TransportServer
metadata:
name: passthrough
spec:
listener:
name: tls-passthrough
protocol: TLS_PASSTHROUGH
host: app.example.com
upstreams:
- name: secure-app
service: secure-app
port: 8443
action:
pass: secure-app
14 changes: 14 additions & 0 deletions tests/data/prometheus/transport-server/tcp.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: k8s.nginx.org/v1alpha1
kind: TransportServer
metadata:
name: tcp
spec:
listener:
name: tcp-server
protocol: TCP
upstreams:
- name: tcp-app
service: tcp-service
port: 3333
action:
pass: tcp-app
14 changes: 14 additions & 0 deletions tests/data/prometheus/transport-server/udp.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: k8s.nginx.org/v1alpha1
kind: TransportServer
metadata:
name: udp
spec:
listener:
name: udp-server
protocol: UDP
upstreams:
- name: udp-app
service: udp-service
port: 3333
action:
pass: udp-app
Loading
0