8000 Collect count of VirtualServer, VirtualServerRoute and TransportServe… · nginx/kubernetes-ingress@1eea001 · GitHub
[go: up one dir, main page]

Skip to content

Commit 1eea001

Browse files
authored
Collect count of VirtualServer, VirtualServerRoute and TransportServer resources (#5095)
1 parent ab4037d commit 1eea001

File tree

10 files changed

+631
-218
lines changed

10 files changed

+631
-218
lines changed

cmd/nginx-ingress/main.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,12 @@ func main() {
152152
controllerNamespace := os.Getenv("POD_NAMESPACE")
153153

154154
transportServerValidator := cr_validation.NewTransportServerValidator(*enableTLSPassthrough, *enableSnippets, *nginxPlus)
155-
virtualServerValidator := cr_validation.NewVirtualServerValidator(cr_validation.IsPlus(*nginxPlus), cr_validation.IsDosEnabled(*appProtectDos), cr_validation.IsCertManagerEnabled(*enableCertManager), cr_validation.IsExternalDNSEnabled(*enableExternalDNS))
155+
virtualServerValidator := cr_validation.NewVirtualServerValidator(
156+
cr_validation.IsPlus(*nginxPlus),
157+
cr_validation.IsDosEnabled(*appProtectDos),
158+
cr_validation.IsCertManagerEnabled(*enableCertManager),
159+
cr_validation.IsExternalDNSEnabled(*enableExternalDNS),
160+
)
156161

157162
if *enableServiceInsight {
158163
createHealthProbeEndpoint(kubeClient, plusClient, cnf)

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
github.com/prometheus/common v0.47.0
2020
github.com/spiffe/go-spiffe/v2 v2.1.7
2121
github.com/stretchr/testify v1.8.4
22+
go.opentelemetry.io/otel v1.21.0
2223
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b
2324
k8s.io/api v0.29.2
2425
k8s.io/apimachinery v0.29.2
@@ -98,7 +99,6 @@ require (
9899
go.etcd.io/etcd/client/v3 v3.5.11 // indirect
99100
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
100101
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
101-
go.opentelemetry.io/otel v1.21.0 // indirect
102102
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
103103
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
104104
go.opentelemetry.io/otel/metric v1.21.0 // indirect

internal/configs/configurator.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,7 @@ func (cnf *Configurator) deleteTransportServer(key string) error {
962962
name := getFileNameForTransportServerFromKey(key)
963963
cnf.nginxManager.DeleteStreamConfig(name)
964964

965+
delete(cnf.transportServers, name)
965966
// update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer
966967
if _, exists := cnf.tlsPassthroughPairs[key]; exists {
967968
delete(cnf.tlsPassthroughPairs, key)
@@ -1468,23 +1469,29 @@ func (cnf *Configurator) GetIngressCounts() map[string]int {
14681469
}
14691470
}
14701471

1471-
for _, min := range cnf.minions {
1472< 6D40 code class="diff-text syntax-highlighted-line deletion">-
counters["minion"] += len(min)
1472+
for _, minion := range cnf.minions {
1473+
counters["minion"] += len(minion)
14731474
}
14741475

14751476
return counters
14761477
}
14771478

1478-
// GetVirtualServerCounts returns the total count of VS/VSR resources that are handled by the Ingress Controller
1479+
// GetVirtualServerCounts returns the total count of
1480+
// VirtualServer and VirtualServerRoute resources that are handled by the Ingress Controller
14791481
func (cnf *Configurator) GetVirtualServerCounts() (vsCount int, vsrCount int) {
14801482
vsCount = len(cnf.virtualServers)
14811483
for _, vs := range cnf.virtualServers {
14821484
vsrCount += len(vs.VirtualServerRoutes)
14831485
}
1484-
14851486
return vsCount, vsrCount
14861487
}
14871488

1489+
// GetTransportServerCounts returns the total count of
1490+
// TransportServer resources that are handled by the Ingress Controller
1491+
func (cnf *Configurator) GetTransportServerCounts() (tsCount int) {
1492+
return len(cnf.transportServers)
1493+
}
1494+
14881495
// AddOrUpdateSpiffeCerts writes Spiffe certs and keys to disk and reloads NGINX
14891496
func (cnf *Configurator) AddOrUpdateSpiffeCerts(svidResponse *workloadapi.X509Context) error {
14901497
svid := svidResponse.DefaultSVID()

internal/k8s/controller.go

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
245245
isLatencyMetricsEnabled: input.IsLatencyMetricsEnabled,
246246
isIPV6Disabled: input.IsIPV6Disabled,
247247
}
248-
249248
eventBroadcaster := record.NewBroadcaster()
250249
eventBroadcaster.StartLogging(glog.Infof)
251250
eventBroadcaster.StartRecordingToSink(&core_v1.EventSinkImpl{
@@ -277,18 +276,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
277276
lbc.externalDNSController = ed_controller.NewController(ed_controller.BuildOpts(context.TODO(), lbc.namespaceList, lbc.recorder, lbc.confClient, input.ResyncPeriod, isDynamicNs))
278277
}
279278

280-
// NIC Telemetry Reporting
281-
if input.EnableTelemetryReporting {
282-
lbc.telemetryChan = make(chan struct{})
283-
collector, err := telemetry.NewCollector(
284-
telemetry.WithTimePeriod(input.TelemetryReportingPeriod),
285-
)
286-
if err != nil {
287-
glog.Fatalf("failed to initialize telemetry collector: %v", err)
288-
}
289-
lbc.telemetryCollector = collector
290-
}
291-
292279
glog.V(3).Infof("Nginx Ingress Controller has class: %v", input.IngressClass)
293280

294281
lbc.namespacedInformers = make(map[string]*namespacedInformer)
@@ -357,6 +344,24 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
357344

358345
lbc.secretStore = secrets.NewLocalSecretStore(lbc.configurator)
359346

347+
// NIC Telemetry Reporting
348+
if input.EnableTelemetryReporting {
349+
collectorConfig := telemetry.CollectorConfig{
350+
K8sClientReader: input.KubeClient,
351+
CustomK8sClientReader: input.ConfClient,
352+
Period: 5 * time.Second,
353+
Configurator: lbc.configurator,
354+
}
355+
lbc.telemetryChan = make(chan struct{})
356+
collector, err := telemetry.NewCollector(
357+
collectorConfig,
358+
)
359+
if err != nil {
360+
glog.Fatalf("failed to initialize telemetry collector: %v", err)
361+
}
362+
lbc.telemetryCollector = collector
363+
}
364+
360365
return lbc
361366
}
362367

internal/k8s/controller_test.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3757,6 +3757,7 @@ func TestNewTelemetryCollector(t *testing.T) {
37573757
testCases := []struct {
37583758
testCase string
37593759
input NewLoadBalancerControllerInput
3760+
collectorConfig telemetry.CollectorConfig
37603761
expectedCollector telemetry.Collector
37613762
}{
37623763
{
@@ -3767,8 +3768,10 @@ func TestNewTelemetryCollector(t *testing.T) {
37673768
TelemetryReportingPeriod: "24h",
37683769
},
37693770
expectedCollector: telemetry.Collector{
3770-
Period: 24 * time.Hour,
3771-
Exporter: telemetry.DiscardExporter,
3771+
Config: telemetry.CollectorConfig{
3772+
Period: 24 * time.Hour,
3773+
},
3774+
Exporter: &telemetry.StdoutExporter{},
37723775
},
37733776
},
37743777
{
@@ -3784,7 +3787,7 @@ func TestNewTelemetryCollector(t *testing.T) {
37843787
for _, tc := range testCases {
37853788
lbc := NewLoadBalancerController(tc.input)
37863789
if reflect.DeepEqual(tc.expectedCollector, lbc.telemetryCollector) {
3787-
t.Fatalf("Expected %x, but got %x", tc.expectedCollector, lbc.telemetryCollector)
3790+
t.Fatalf("Expected %v, but got %v", tc.expectedCollector, lbc.telemetryCollector)
37883791
}
37893792
}
37903793
}

internal/telemetry/collector.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// Package telemetry provides functionality for collecting and exporting NIC telemetry data.
2+
package telemetry
3+
4+
import (
5+
"context"
6+
"io"
7+
"time"
8+
9+
"github.com/nginxinc/kubernetes-ingress/internal/configs"
10+
11+
k8s_nginx "github.com/nginxinc/kubernetes-ingress/pkg/client/clientset/versioned"
12+
"k8s.io/apimachinery/pkg/util/wait"
13+
"k8s.io/client-go/kubernetes"
14+
15+
"github.com/golang/glog"
16+
)
17+
18+
// Option is a functional option used for configuring TraceReporter.
19+
type Option func(*Collector) error
20+
21+
// WithExporter configures telemetry collector to use given exporter.
22+
//
23+
// This may change in the future when we use exporter implemented
24+
// in the external module.
25+
func WithExporter(e Exporter) Option {
26+
return func(c *Collector) error {
27+
c.Exporter = e
28+
return nil
29+
}
30+
}
31+
32+
// Collector is NIC telemetry data collector.
33+
type Collector struct {
34+
// Exporter is a temp exporter for exporting telemetry data.
35+
// The concrete implementation will be implemented in a separate module.
36+
Exporter Exporter
37+
38+
// Configuration for the collector.
39+
Config CollectorConfig
40+
}
41+
42+
// CollectorConfig contains configuration options for a Collector
43+
type CollectorConfig struct {
44+
// K8sClientReader is a kubernetes client.
45+
K8sClientReader kubernetes.Interface
46+
47+
// CustomK8sClientReader is a kubernetes client for our CRDs.
48+
// Note: May not need this client.
49+
CustomK8sClientReader k8s_nginx.Interface
50+
51+
// Period to collect telemetry
52+
Period time.Duration
53+
54+
Configurator *configs.Configurator
55+
}
56+
57+
// NewCollector takes 0 or more options and creates a new TraceReporter.
58+
// If no options are provided, NewReporter returns TraceReporter
59+
// configured to gather data every 24h.
60+
func NewCollector(cfg CollectorConfig, opts ...Option) (*Collector, error) {
61+
c := Collector{
62+
Exporter: &StdoutExporter{Endpoint: io.Discard},
63+
Config: cfg,
64+
}
65+
for _, o := range opts {
66+
if err := o(&c); err != nil {
67+
return nil, err
68+
}
69+
}
70+
return &c, nil
71+
}
72+
73+
// Start starts running NIC Telemetry Collector.
74+
func (c *Collector) Start(ctx context.Context) {
75+
wait.JitterUntilWithContext(ctx, c.Collect, c.Config.Period, 0.1, true)
76+
}
77+
78+
// Collect collects and exports telemetry data.
79+
// It exports data using provided exporter.
80+
func (c *Collector) Collect(ctx context.Context) {
81+
glog.V(3).Info("Collecting telemetry data")
82+
// TODO: Re-add ctx to BuildReport when collecting Node Count.
83+
data, err := c.BuildReport()
84+
if err != nil {
85+
glog.Errorf("Error collecting telemetry data: %v", err)
86+
}
87+
err = c.Exporter.Export(ctx, data)
88+
if err != nil {
89+
glog.Errorf("Error exporting telemetry data: %v", err)
90+
}
91+
glog.V(3).Infof("Exported telemetry data: %+v", data)
92+
}
93+
94+
// BuildReport takes context and builds report from gathered telemetry data.
95+
func (c *Collector) BuildReport() (Data, error) {
96+
d := Data{}
97+
var err error
98+
99+
if c.Config.Configurator != nil {
100+
d.NICResourceCounts.VirtualServers, d.NICResourceCounts.VirtualServerRoutes = c.Config.Configurator.GetVirtualServerCounts()
101+
d.NICResourceCounts.TransportServers = c.Config.Configurator.GetTransportServerCounts()
102+
}
103+
return d, err
104+
}

0 commit comments

Comments
 (0)
0