8000 add license reporting process (#6309) · nginx/kubernetes-ingress@bfcbf4c · GitHub
[go: up one dir, main page]
Skip to content

Commit bfcbf4c

Browse files
author
Jim Ryan
authored
add license reporting process (#6309)
1 parent 4a88378 commit bfcbf4c

File tree

8 files changed

+289
-48
lines changed

8 files changed

+289
-48
lines changed

‎build/scripts/common.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ if [ -z "${BUILD_OS##*plus*}" ]; then
77
mkdir -p /etc/nginx/oidc/
88
cp -a /code/internal/configs/oidc/* /etc/nginx/oidc/
99
mkdir -p /etc/nginx/state_files/
10+
mkdir -p /etc/nginx/reporting/
1011
PLUS=-plus
1112
fi
1213

‎cmd/nginx-ingress/main.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/nginxinc/kubernetes-ingress/internal/healthcheck"
2222
"github.com/nginxinc/kubernetes-ingress/internal/k8s"
2323
"github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets"
24+
license_reporting "github.com/nginxinc/kubernetes-ingress/internal/license_reporting"
2425
"github.com/nginxinc/kubernetes-ingress/internal/metrics"
2526
"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
2627
"github.com/nginxinc/kubernetes-ingress/internal/nginx"
@@ -41,6 +42,7 @@ import (
4142
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
4243

4344
kitlog "github.com/go-kit/log"
45+
4446
"github.com/go-kit/log/level"
4547

4648
nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
@@ -96,7 +98,13 @@ func main() {
9698

9799
managerCollector, controllerCollector, registry := createManagerAndControllerCollectors(ctx, constLabels)
98100

99-
nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector)
101+
var licenseReporter *license_reporting.LicenseReporter
102+
103+
if *nginxPlus {
104+
licenseReporter = license_reporting.NewLicenseReporter(kubeClient)
105+
}
106+
107+
nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector, licenseReporter)
100108

101109
nginxVersion := getNginxVersionInfo(ctx, nginxManager)
102110

@@ -452,14 +460,14 @@ func createTemplateExecutors(ctx context.Context) (*version1.TemplateExecutor, *
452460
return templateExecutor, templateExecutorV2
453461
}
454462

455-
func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector) (nginx.Manager, bool) {
463+
func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector, licenseReporter *license_reporting.LicenseReporter) (nginx.Manager, bool) {
456464
useFakeNginxManager := *proxyURL != ""
457465
var nginxManager nginx.Manager
458466
if useFakeNginxManager {
459467
nginxManager = nginx.NewFakeManager("/etc/nginx")
460468
} else {
461469
timeout := time.Duration(*nginxReloadTimeout) * time.Millisecond
462-
nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, timeout)
470+
nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, licenseReporter, timeout, *nginxPlus)
463471
}
464472
return nginxManager, useFakeNginxManager
465473
}

‎internal/common_cluster_info/common_cluster_info.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package commonclusterinfo
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8+
"k8s.io/apimachinery/pkg/types"
9+
"k8s.io/client-go/kubernetes"
10+
)
11+
12+
// This file contains functions for data used in both product telemetry and license reporting
13+
14+
// GetNodeCount returns the number of nodes in the cluster
15+
func GetNodeCount(ctx context.Context, client kubernetes.Interface) (int, error) {
16+
nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
17+
if err != nil {
18+
return 0, err
19+
}
20+
return len(nodes.Items), nil
21+
}
22+
23+
// GetClusterID returns the UID of the kube-system namespace representing cluster id.
24+
// It returns an error if the underlying k8s API client errors.
25+
func GetClusterID(ctx context.Context, client kubernetes.Interface) (string, error) {
26+
cluster, err := client.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{})
27+
if err != nil {
28+
return "", err
29+
}
30+
return string(cluster.UID), nil
31+
}
32+
33+
// GetInstallationID returns the Installation ID of the cluster
34+
func GetInstallationID(ctx context.Context, client kubernetes.Interface, podNSName types.NamespacedName) (_ string, err error) {
35+
defer func() {
36+
if err != nil {
37+
err = fmt.Errorf("error generating InstallationID: %w", err)
38+
}
39+
}()
40+
41+
pod, err := client.CoreV1().Pods(podNSName.Namespace).Get(ctx, podNSName.Name, metav1.GetOptions{})
42+
if err != nil {
43+
return "", err
44+
}
45+
podOwner := pod.GetOwnerReferences()
46+
if len(podOwner) != 1 {
47+
return "", fmt.Errorf("expected pod owner reference to be 1, got %d", len(podOwner))
48+
}
49+
50+
switch podOwner[0].Kind {
51+
case "ReplicaSet":
52+
rs, err := client.AppsV1().ReplicaSets(podNSName.Namespace).Get(ctx, podOwner[0].Name, metav1.GetOptions{})
53+
if err != nil {
54+
return "", err
55+
}
56+
rsOwner := rs.GetOwnerReferences() // rsOwner holds information about replica's owner - Deployment object
57+
if len(rsOwner) != 1 {
58+
return "", fmt.Errorf("expected replicaset owner reference to be 1, got %d", len(rsOwner))
59+
}
60+
return string(rsOwner[0].UID), nil
61+
case "DaemonSet":
62+
return string(podOwner[0].UID), nil
63+
default:
64+
return "", fmt.Errorf("expected pod owner reference to be ReplicaSet or DeamonSet, got %s", podOwner[0].Kind)
65+
}
66+
}

‎internal/configs/version1/nginx-plus.tmpl‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -358,8 +358,15 @@ stream {
358358
include /etc/nginx/stream-conf.d/*.conf;
359359
}
360360

361-
{{- if (.NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r31") }}
361+
{{- if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r33" }}
362+
mgmt {
363+
usage_report endpoint=product.connect.nginx.com interval=1h;
364+
uuid_file /var/lib/nginx/nginx.id;
365+
license_token conf/license.jwt;
366+
deployment_context /etc/nginx/reporting/tracking.info;
367+
}
368+
{{- else if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r31" }}
362369
mgmt {
363370
usage_report interval=0s;
364371
}
365-
{{- end}}
372+
{{- end }}

‎internal/license_reporting/license_reporting.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package licensereporting
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"log/slog"
7+
"os"
8+
"path/filepath"
9+
"time"
10+
11+
nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
12+
13+
clusterInfo "github.com/nginxinc/kubernetes-ingress/internal/common_cluster_info"
14+
"k8s.io/apimachinery/pkg/types"
15+
"k8s.io/apimachinery/pkg/util/wait"
16+
"k8s.io/client-go/kubernetes"
17+
)
18+
19+
var (
20+
reportingDir = "/etc/nginx/reporting"
21+
reportingFile = "tracking.info"
22+
)
23+
24+
type licenseInfo struct {
25+
Integration string `json:"integration"`
26+
ClusterID string `json:"cluster_id"`
27+
ClusterNodeCount int `json:"cluster_node_count"`
28+
InstallationID string `json:"installation_id"`
29+
}
30+
31+
func newLicenseInfo(clusterID, installationID string, clusterNodeCount int) *licenseInfo {
32+
return &licenseInfo{
33+
Integration: "nic",
34+
ClusterID: clusterID,
35+
InstallationID: installationID,
36+
ClusterNodeCount: clusterNodeCount,
37+
}
38+
}
39+
40+
func writeLicenseInfo(l *slog.Logger, info *licenseInfo) {
41+
jsonData, err := json.Marshal(info)
42+
if err != nil {
43+
nl.Errorf(l, "failed to marshal LicenseInfo to JSON: %v", err)
44+
return
45+
}
46+
filePath := filepath.Join(reportingDir, reportingFile)
47+
if err := os.WriteFile(filePath, jsonData, 0o600); err != nil {
48+
nl.Errorf(l, "failed to write license reporting info to file: %v", err)
49+
}
50+
}
51+
52+
// LicenseReporter can start the license reporting process
53+
type LicenseReporter struct {
54+
config LicenseReporterConfig
55+
}
56+
57+
// LicenseReporterConfig contains the information needed for license reporting
58+
type LicenseReporterConfig struct {
59+
Period time.Duration
60+
K8sClientReader kubernetes.Interface
61+
PodNSName types.NamespacedName
62+
}
63+
64+
// NewLicenseReporter creates a new LicenseReporter
65+
func NewLicenseReporter(client kubernetes.Interface) *LicenseReporter {
66+
return &LicenseReporter{
67+
config: LicenseReporterConfig{
68+
Period: 24 * time.Hour,
69+
K8sClientReader: client,
70+
PodNSName: types.NamespacedName{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("POD_NAME")},
71+
},
72+
}
73+
}
74+
75+
// Start begins the license report writer process for NIC
76+
func (lr *LicenseReporter) Start(ctx context.Context) {
77+
wait.JitterUntilWithContext(ctx, lr.collectAndWrite, lr.config.Period, 0.1, true)
78+
}
79+
80+
func (lr *LicenseReporter) collectAndWrite(ctx context.Context) {
81+
l := nl.LoggerFromContext(ctx)
82+
clusterID, err := clusterInfo.GetClusterID(ctx, lr.config.K8sClientReader)
83+
if err != nil {
84+
nl.Errorf(l, "Error collecting ClusterIDS: %v", err)
85+
}
86+
nodeCount, err := clusterInfo.GetNodeCount(ctx, lr.config.K8sClientReader)
87+
if err != nil {
88+
nl.Errorf(l, "Error collecting ClusterNodeCount: %v", err)
89+
}
90+
installationID, err := clusterInfo.GetInstallationID(ctx, lr.config.K8sClientReader, lr.config.PodNSName)
91+
if err != nil {
92+
nl.Errorf(l, "Error collecting InstallationID: %v", err)
93+
}
94+
info := newLicenseInfo(clusterID, installationID, nodeCount)
95+
writeLicenseInfo(l, info)
96+
}

‎internal/license_reporting/license_reporting_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package licensereporting
2+
3+
import (
4+
"encoding/json"
5+
"io"
6+
"log/slog"
7+
"os"
8+
"path/filepath"
9+
"testing"
10+
11+
nic_glog "github.com/nginxinc/kubernetes-ingress/internal/logger/glog"
12+
"github.com/nginxinc/kubernetes-ingress/internal/logger/levels"
13+
14+
"k8s.io/client-go/kubernetes/fake"
15+
)
16+
17+
func TestNewLicenseInfo(t *testing.T) {
18+
info := newLicenseInfo("test-cluster", "test-installation", 5)
19+
20+
if info.Integration != "nic" {
21+
t.Errorf("newLicenseInfo() Integration = %v, want %v", info.Integration, "nic")
22+
}
23+
if info.ClusterID != "test-cluster" {
24+
t.Errorf("newLicenseInfo() ClusterID = %v, want %v", info.ClusterID, "test-cluster")
25+
}
26+
if info.InstallationID != "test-installation" {
27+
t.Errorf("newLicenseInfo() InstallationID = %v, want %v", info.InstallationID, "test-installation")
28+
}
29+
if info.ClusterNodeCount != 5 {
30+
t.Errorf("newLicenseInfo() ClusterNodeCount = %v, want %v", info.ClusterNodeCount, 5)
31+
}
32+
}
33+
34+
func TestWriteLicenseInfo(t *testing.T) {
35+
tempDir := t.TempDir()
36+
oldReportingDir := reportingDir
37+
reportingDir = tempDir
38+
defer func() { reportingDir = oldReportingDir }()
39+
40+
l := slog.New(nic_glog.New(io.Discard, &nic_glog.Options{Level: levels.LevelInfo}))
41+
info := newLicenseInfo("test-cluster", "test-installation", 5)
42+
writeLicenseInfo(l, info)
43+
44+
filePath := filepath.Join(tempDir, reportingFile)
45+
if _, err := os.Stat(filePath); os.IsNotExist(err) {
46+
t.Fatalf("Expected file %s to exist, but it doesn't", filePath)
47+
}
48+
49+
/* #nosec G304 */
50+
content, err := os.ReadFile(filePath)
51+
if err != nil {
52+
t.Fatalf("Failed to read file: %v", err)
53+
}
54+
55+
var readInfo licenseInfo
56+
err = json.Unmarshal(content, &readInfo)
57+
if err != nil {
58+
t.Fatalf("Failed to unmarshal JSON: %v", err)
59+
}
60+
61+
if readInfo != *info {
62+
t.Errorf("Written info does not match original. Got %+v, want %+v", readInfo, *info)
63+
}
64+
}
65+
66+
func TestNewLicenseReporter(t *testing.T) {
67+
reporter := NewLicenseReporter(fake.NewSimpleClientset())
68+
if reporter == nil {
69+
t.Fatal("NewLicenseReporter() returned nil")
70+
}
71+
}

‎internal/nginx/manager.go‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"strings"
1515
"time"
1616

17+
license_reporting "github.com/nginxinc/kubernetes-ingress/internal/license_reporting"
1718
nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
1819
"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
1920

@@ -101,7 +102,7 @@ type Manager interface {
101102
DeleteKeyValStateFiles(virtualServerName string)
102103
}
103104

104-
// LocalManager updates NGINX configuration, starts, reloads and quits NGINX,
105+
// LocalManager updates NGINX configuration, starts, reloads and quits NGINX, updates License Reporting file
105106
// updates NGINX Plus upstream servers. It assumes that NGINX is running in the same container.
106107
type LocalManager struct {
107108
confdPath string
@@ -119,15 +120,18 @@ type LocalManager struct {
119120
plusClient *client.NginxClient
120121
plusConfigVersionCheckClient *http.Client
121122
metricsCollector collectors.ManagerCollector
123+
licenseReporter *license_reporting.LicenseReporter
124+
licenseReporterCancel context.CancelFunc
122125
OpenTracing bool
123126
appProtectPluginPid int
124127
appProtectDosAgentPid int
125128
agentPid int
126129
logger *slog.Logger
130+
nginxPlus bool
127131
}
128132

129133
// NewLocalManager creates a LocalManager.
130-
func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration) *LocalManager {
134+
func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, lr *license_reporting.LicenseReporter, timeout time.Duration, nginxPlus bool) *LocalManager {
131135
l := nl.LoggerFromContext(ctx)
132136
verifyConfigGenerator, err := newVerifyConfigGenerator()
133137
if err != nil {
@@ -148,6 +152,8 @@ func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collec
148152
configVersion: 0,
149153
verifyClient: newVerifyClient(timeout),
150154
metricsCollector: mc,
155+
licenseReporter: lr,
156+
nginxPlus: nginxPlus,
151157
logger: l,
152158
}
153159

@@ -296,6 +302,19 @@ func (lm *LocalManager) ClearAppProtectFolder(name string) {
296302

297303
// Start starts NGINX.
298304
func (lm *LocalManager) Start(done chan error) {
305+
if lm.nginxPlus {
306+
isR33OrGreater, versionErr := lm.Version().PlusGreaterThanOrEqualTo("nginx-plus-r33")
307+
if versionErr != nil {
308+
nl.Errorf(lm.logger, "Error determining whether nginx version is >= r33: %v", versionErr)
309+
}
310+
if isR33OrG 38BA reater {
311+
ctx, cancel := context.WithCancel(context.Background())
312+
nl.ContextWithLogger(ctx, lm.logger)
313+
go lm.licenseReporter.Start(ctx)
314+
lm.licenseReporterCancel = cancel
315+
}
316+
}
317+
299318
nl.Debug(lm.logger, "Starting nginx")
300319

301320
binaryFilename := getBinaryFileName(lm.debug)
@@ -347,6 +366,16 @@ func (lm *LocalManager) Reload(isEndpointsUpdate bool) error {
347366
func (lm *LocalManager) Quit() {
348367
nl.Debugf(lm.logger, "Quitting nginx")
349368

369+
if lm.nginxPlus {
370+
isR33OrGreater, err := lm.Version().PlusGreaterThanOrEqualTo("nginx-plus-r33")
371+
if err != nil {
372+
nl.Errorf(lm.logger, "Error determining whether nginx version is >= r33: %v", err)
373+
}
374+
if isR33OrGreater && lm.licenseReporterCancel != nil {
375+
lm.licenseReporterCancel()
376+
}
377+
}
378+
350379
binaryFilename := getBinaryFileName(lm.debug)
351380
if err := shellOut(lm.logger, fmt.Sprintf("%v -s %v", binaryFilename, "quit")); err != nil {
352381
nl.Fatalf(lm.logger, "Failed to quit nginx: %v", err)

0 commit comments

Comments
 (0)
0