8000 StatefulSet support (#8159) · nginx/kubernetes-ingress@d3dcfe2 · GitHub
[go: up one dir, main page]
Skip to content

Commit d3dcfe2

Browse files
haywoodshvepatel
authored and
AlexFenlon
committed
StatefulSet support (#8159)
Introduce StatefulSet deployment support for the NGINX Ingress Controller to address cache persistence use cases. When deployed as a StatefulSet, NIC can maintain persistent volumes that survive pod replacements, ensuring cache data is preserved during normal Kubernetes lifecycle operations like pod rebalancing. - add complete StatefulSet template with nginx-cache volumeClaimTemplate - automatic `nginx-cache volume` provisioning, creating one persistent volume claim specifically for nginx cache storage, mounted at `/var/cache/nginx` - full compatibility with `readOnlyRootFilesystem` - automatically create headless service for `zone-sync` - update schema with StatefulSet-specific configuration options including persistent volume claim retention policies - update integration tests to support StatefulSet - add helm tests Configuration Options: ``` controller: statefulset: podManagementPolicy: "OrderedReady" persistentVolumeClaimRetentionPolicy: whenDeleted: "Retain" whenScaled: "Retain" nginxCachePVC: size: "256Mi" storageClass: "standard-rwo" # if not provided, use cluster default accessModes: - "ReadWriteOnce" ``` --------- Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Co-authored-by: Venktesh <ve.patel@f5.com>
1 parent 679d3d3 commit d3dcfe2

25 files changed

+4763
-1037
lines changed

‎charts/nginx-ingress/templates/_helpers.tpl‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -392,14 +392,17 @@ List of volumes for controller.
392392
{{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }}
393393
- name: nginx-etc
394394
emptyDir: {}
395-
- name: nginx-cache
396-
emptyDir: {}
397395
- name: nginx-lib
398396
emptyDir: {}
399397
- name: nginx-state
400398
emptyDir: {}
401399
- name: nginx-log
402400
emptyDir: {}
401+
{{- /* For StatefulSet, nginx-cache volume is always provided via volumeClaimTemplates */ -}}
402+
{{- if ne .Values.controller.kind "statefulset" }}
403+
- name: nginx-cache
404+
emptyDir: {}
405+
{{- end }}
403406
{{- end }}
404407
{{- if .Values.controller.appprotect.v5 }}
405408
{{ toYaml .Values.controller.appprotect.volumes }}
@@ -459,6 +462,9 @@ volumeMounts:
459462
name: nginx-state
460463
- mountPath: /var/log/nginx
461464
name: nginx-log
465+
{{- else if eq .Values.controller.kind "statefulset" }}
466+
- mountPath: /var/cache/nginx
467+
name: nginx-cache
462468
{{- end }}
463469
{{- if .Values.controller.appprotect.v5 }}
464470
- name: app-protect-bd-config

‎charts/nginx-ingress/templates/clusterrole.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ rules:
7171
resources:
7272
- replicasets
7373
- daemonsets
74+
- statefulsets
7475
verbs:
7576
- get
7677
- apiGroups:

‎charts/nginx-ingress/templates/controller-statefulset.yaml‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
{{- if eq .Values.controller.kind "statefulset" }}
2+
apiVersion: apps/v1
3+
kind: StatefulSet
4+
metadata:
5+
name: {{ include "nginx-ingress.controller.fullname" . }}
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "nginx-ingress.labels" . | nindent 4 }}
9+
{{- if .Values.controller.annotations }}
10+
annotations: {{ toYaml .Values.controller.annotations | nindent 4 }}
11+
{{- end }}
12+
spec:
13+
{{- if not .Values.controller.autoscaling.enabled }}
14+
replicas: {{ .Values.controller.replicaCount }}
15+
{{- end }}
16+
serviceName: {{ include "nginx-ingress.controller.service.name" . }}
17+
selector:
18+
matchLabels:
19+
{{- include "nginx-ingress.selectorLabels" . | nindent 6 }}
20+
template:
21+
metadata:
22+
labels:
23+
{{- include "nginx-ingress.podLabels" . | nindent 8 }}
24+
{{- if or .Values.prometheus.create .Values.controller.pod.annotations }}
25+
annotations:
26+
{{- if .Values.prometheus.create }}
27+
prometheus.io/scrape: "true"
28+
prometheus.io/port: "{{ .Values.prometheus.port }}"
29+
prometheus.io/scheme: "{{ .Values.prometheus.scheme }}"
30+
{{- end }}
31+
{{- if .Values.controller.pod.annotations }}
32+
{{ toYaml .Values.controller.pod.annotations | indent 8 }}
33+
{{- end }}
34+
{{- end }}
35+
spec:
36+
{{- if .Values.controller.nodeSelector }}
37+
nodeSelector:
38+
{{ toYaml .Values.controller.nodeSelector | indent 8 }}
39+
{{- end }}
40+
{{- if .Values.controller.tolerations }}
41+
tolerations:
42+
{{ toYaml .Values.controller.tolerations | indent 6 }}
43+
{{- end }}
44+
{{- if .Values.controller.affinity }}
45+
affinity:
46+
{{ toYaml .Values.controller.affinity | indent 8 }}
47+
{{- end }}
48+
{{- if .Values.controller.topologySpreadConstraints }}
49+
topologySpreadConstraints:
50+
{{ toYaml .Values.controller.topologySpreadConstraints | indent 8 }}
51+
{{- end }}
52+
{{- include "nginx-ingress.volumes" . | indent 6 }}
53+
{{- if .Values.controller.priorityClassName }}
54+
priorityClassName: {{ .Values.controller.priorityClassName }}
55+
{{- end }}
56+
serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }}
57+
automountServiceAccountToken: true
58+
securityContext:
59+
{{ toYaml .Values.controller.podSecurityContext | indent 8 }}
60+
terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
61+
hostNetwork: {{ .Values.controller.hostNetwork }}
62+
dnsPolicy: {{ .Values.controller.dnsPolicy }}
63+
{{- if .Values.controller.shareProcessNamespace }}
64+
shareProcessNamespace: true
65+
{{- end }}
66+
containers:
67+
- image: {{ include "nginx-ingress.image" . }}
68+
name: {{ include "nginx-ingress.name" . }}
69+
imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}"
70+
{{- if .Values.controller.lifecycle }}
71+
lifecycle:
72+
{{ toYaml .Values.controller.lifecycle | indent 10 }}
73+
{{- end }}
74+
ports:
75+
{{- range $key, $value := .Values.controller.containerPort }}
76+
- name: {{ $key }}
77+
containerPort: {{ $value }}
78+
protocol: TCP
79+
{{- if and $.Values.controller.hostPort.enable (index $.Values.controller.hostPort $key) }}
80+
hostPort: {{ index $.Values.controller.hostPort $key }}
81+
{{- end }}
82+
{{- end }}
83+
{{- if .Values.controller.customPorts }}
84+
{{ toYaml .Values.controller.customPorts | indent 8 }}
85+
{{- end }}
86+
{{- if .Values.prometheus.create }}
87+
- name: prometheus
88+
containerPort: {{ .Values.prometheus.port }}
89+
{{- end }}
90+
{{- if .Values.serviceInsight.create }}
91+
- name: service-insight
92+
containerPort: {{ .Values.serviceInsight.port }}
93+
{{- end }}
94+
{{- if .Values.controller.readyStatus.enable }}
95+
- name: readiness-port
96+
containerPort: {{ .Values.controller.readyStatus.port }}
97+
{{- end }}
98+
{{- if .Values.controller.startupStatus.enable }}
99+
- name: startup-port
100+
containerPort: {{ .Values.controller.startupStatus.port }}
101+
{{- end }}
102+
{{- if .Values.controller.readyStatus.enable }}
103+
readinessProbe:
104+
httpGet:
105+
path: /nginx-ready
106+
port: readiness-port
107+
periodSeconds: 1
108+
initialDelaySeconds: {{ .Values.controller.readyStatus.initialDelaySeconds }}
109+
{{- end }}
110+
{{- if .Values.controller.startupStatus.enable }}
111+
startupProbe:
112+
httpGet:
113+
path: {{ .Values.controller.startupStatus.path }}
114+
port: startup-port
115+
initialDelaySeconds: {{ .Values.controller.startupStatus.initialDelaySeconds }}
116+
periodSeconds: {{ .Values.controller.startupStatus.periodSeconds }}
117+
timeoutSeconds: {{ .Values.controller.startupStatus.timeoutSeconds }}
118+
successThreshold: {{ .Values.controller.startupStatus.successThreshold }}
119+
failureThreshold: {{ .Values.controller.startupStatus.failureThreshold }}
120+
{{- end }}
121+
resources:
122+
{{ toYaml .Values.controller.resources | indent 10 }}
123+
{{- if .Values.controller.securityContext }}
124+
securityContext:
125+
{{ toYaml .Values.controller.securityContext | indent 10 }}
126+
{{- else }}
127+
securityContext:
128+
allowPrivilegeEscalation: false
129+
readOnlyRootFilesystem: {{ .Values.controller.readOnlyRootFilesystem }}
130+
runAsUser: 101 #nginx
131+
runAsNonRoot: true
132+
capabilities:
133+
drop:
134+
- ALL
135+
add:
136+
- NET_BIND_SERVICE
137+
{{- end }}
138+
{{- include "nginx-ingress.volumeMounts" . | indent 8 }}
139+
env:
140+
- name: POD_NAMESPACE
141+
valueFrom:
142+
fieldRef:
143+
fieldPath: metadata.namespace
144+
- name: POD_NAME
145+
valueFrom:
146+
fieldRef:
147+
fieldPath: metadata.name
148+
{{- if .Values.controller.env }}
149+
{{ toYaml .Values.controller.env | indent 8 }}
150+
{{- end }}
151+
{{- if .Values.nginxServiceMesh.enable }}
152+
- name: POD_SERVICEACCOUNT
153+
valueFrom:
154+
fieldRef:
155+
fieldPath: spec.serviceAccountName
156+
{{- end }}
157+
{{- if hasKey .Values.controller.mgmt "usageReport" -}}
158+
{{- if hasKey .Values.controller.mgmt.usageReport "proxyCredentialsSecretName" }}
159+
{{- if not (hasKey .Values.controller.mgmt.usageReport "proxyHost") -}}
160+
{{- fail "Error: 'controller.mgmt.usageReport.proxyHost' must be set when using 'controller.mgmt.usageReport.proxyCredentialsSecretName'." }}
161+
{{- end }}
162+
- name: PROXY_USER
163+
valueFrom:
164+
secretKeyRef:
165+
name: {{ .Values.controller.mgmt.usageReport.proxyCredentialsSecretName }}
166+
key: username
167+
- name: PROXY_PASS
168+
valueFrom:
169+
secretKeyRef:
170+
name: {{ .Values.controller.mgmt.usageReport.proxyCredentialsSecretName }}
171+
key: password
172+
{{- end }}
173+
{{- end }}
174+
args:
175+
{{- include "nginx-ingress.args" . | nindent 10 }}
176+
{{- if .Values.controller.extraContainers }}
177+
{{ toYaml .Values.controller.extraContainers | nindent 6 }}
178+
{{- end }}
179+
180+
{{- include "nginx-ingress.appprotect.v5" . | nindent 6 }}
181+
182+
{{- if or ( eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" ) .Values.controller.initContainers }}
183+
initContainers:
184+
{{- end }}
185+
{{- if eq (include "nginx-ingress.readOnlyRootFilesystem" .) "true" }}
186+
- name: init-{{ include "nginx-ingress.name" . }}
187+
image: {{ include "nginx-ingress.image" . }}
188+
imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}"
189+
command: ['cp', '-vdR', '/etc/nginx/.', '/mnt/etc']
190+
{{- if .Values.controller.initContainerResources }}
191+
resources:
192+
{{ toYaml .Values.controller.initContainerResources | indent 10 }}
193+
{{- end }}
194+
{{- if .Values.controller.initContainerSecurityContext }}
195+
securityContext:
196+
{{ toYaml .Values.controller.initContainerSecurityContext | indent 10 }}
197+
{{- else }}
198+
securityContext:
199+
allowPrivilegeEscalation: false
200+
readOnlyRootFilesystem: true
201+
runAsUser: 101 #nginx
202+
runAsNonRoot: true
203+
capabilities:
204+
drop:
205+
- ALL
206+
{{- end }}
207+
volumeMounts:
208+
- mountPath: /mnt/etc
209+
name: nginx-etc
210+
{{- end }}
211+
{{- if .Values.controller.initContainers }}
212+
{{ toYaml .Values.controller.initContainers | indent 6 }}
213+
{{- end }}
214+
{{- if .Values.controller.strategy }}
215+
updateStrategy:
216+
{{ toYaml .Values.controller.strategy | indent 4 }}
217+
{{- end }}
218+
{{- if .Values.controller.minReadySeconds }}
219+
minReadySeconds: {{ .Values.controller.minReadySeconds }}
220+
{{- end }}
221+
{{- if .Values.controller.statefulset.podManagementPolicy }}
222+
podManagementPolicy: {{ .Values.controller.statefulset.podManagementPolicy }}
223+
{{- end }}
224+
{{- if .Values.controller.statefulset.persistentVolumeClaimRetentionPolicy }}
225+
persistentVolumeClaimRetentionPolicy:
226+
{{ toYaml .Values.controller.statefulset.persistentVolumeClaimRetentionPolicy | indent 4 }}
227+
{{- end }}
228+
volumeClaimTemplates:
229+
- metadata:
230+
name: nginx-cache
231+
spec:
232+
accessModes:
233+
{{ toYaml .Values.controller.statefulset.nginxCachePVC.accessModes | indent 8 }}
234+
{{- if .Values.controller.statefulset.nginxCachePVC.storageClass }}
235+
storageClassName: {{ .Values.controller.statefulset.nginxCachePVC.storageClass | quote }}
236+
{{- end }}
237+
resources:
238+
requests:
239+
storage: {{ .Values.controller.statefulset.nginxCachePVC.size | quote }}
240+
{{- end }}

‎charts/nginx-ingress/values.schema.json‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@
3535
"title": "The kind of the Ingress Controller",
3636
"enum": [
3737
"deployment",
38-
"daemonset"
38+
"daemonset",
39+
"statefulset"
3940
],
4041
"examples": [
4142
"deployment",
42-
"daemonset"
43+
"daemonset",
44+
"statefulset"
4345
]
4446
},
4547
"selectorLabels": {
@@ -994,6 +996,55 @@
994996
}
995997
]
996998
},
999+
"statefulset": {
1000+
"type": "object",
1001+
"default": {},
1002+
"title": "The StatefulSet configuration Schema",
1003+
"properties": {
1004+
"podManagementPolicy": {
1005+
"type": "string",
1006+
"default": "OrderedReady",
1007+
"title": "The pod management policy",
1008+
"enum": [
1009+
"OrderedReady",
1010+
"Parallel"
1011+
]
1012+
},
1013+
"persistentVolumeClaimRetentionPolicy": {
1014+
"type": "object",
1015+
"default": {},
1016+
"title": "The persistentVolumeClaimRetentionPolicy Schema",
1017+
"$ref": "https://raw.githubusercontent.com/nginxinc/kubernetes-json-schema/master/v1.33.1/_definitions.json#/definitions/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy"
1018+
},
1019+
"nginxCachePVC": {
1020+
"type": "object",
1021+
"default": {},
1022+
"title": "The nginxCachePVC Schema",
1023+
"properties": {
1024+
"size": {
1025+
"type": "string",
1026+
"default": "256Mi",
1027+
"title": "The size Schema",
1028+
"$ref": "https://raw.githubusercontent.com/nginxinc/kubernetes-json-schema/master/v1.33.1/_definitions.json#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity"
1029+
},
1030+
"storageClass": {
1031+
"type": "string",
1032+
"title": "The storageClass Schema",
1033+
"default": "",
1034+
"$ref": "https://raw.githubusercontent.com/nginxinc/kubernetes-json-schema/master/v1.33.1/_definitions.json#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimSpec/properties/storageClassName"
1035+
},
1036+
"accessModes": {
1037+
"type": "array",
1038+
"default": [
1039+
"ReadWriteOnce"
1040+
],
1041+
"title": "The accessModes Schema",
1042+
"$ref": "https://raw.githubusercontent.com/nginxinc/kubernetes-json-schema/master/v1.33.1/_definitions.json#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimSpec/properties/accessModes"
1043+
}
1044+
}
1045+
}
1046+
}
1047+
},
9971048
"extraContainers": {
9981049
"type": "array",
9991050
"default": [],

‎charts/nginx-ingress/values.yaml‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
controller:
2-
## The name of the Ingress Controller daemonset or deployment.
2+
## The name of the Ingress Controller daemonset, deployment, or statefulset.
33
name: controller
44

5-
## The kind of the Ingress Controller installation - deployment or daemonset.
5+
## The kind of the Ingress Controller installation - deployment, daemonset, or statefulset.
66
kind: deployment
77

88
## The selectorLabels used to override the default values.
@@ -344,6 +344,30 @@ controller:
344344
## Strategy used to replace old Pods by new ones. .spec.strategy.type can be "Recreate" or "RollingUpdate" for Deployments, and "OnDelete" or "RollingUpdate" for Daemonsets. "RollingUpdate" is the default value.
345345
strategy: {}
346346

347+
## StatefulSet-specific configuration (only used when kind is "statefulset")
348+
statefulset:
349+
## Pod management policy for StatefulSet. Can be "OrderedReady" or "Parallel".
350+
## OrderedReady will start pods one at a time in order, Parallel will start all pods at once.
351+
podManagementPolicy: "OrderedReady"
352+
353+
## PersistentVolumeClaim retention policy for StatefulSet
354+
## Determines when to delete PVCs when the StatefulSet is deleted or scaled down
355+
persistentVolumeClaimRetentionPolicy:
356+
## When to delete PVCs when the StatefulSet is deleted. Can be "Retain" or "Delete".
357+
whenDeleted: "Retain"
358+
## When to delete PVCs when the StatefulSet is scaled down. Can be "Retain" or "Delete".
359+
whenScaled: "Retain"
360+
361+
## Configuration for StatefulSet nginx-cache PVC
362+
nginxCachePVC:
363+
## Storage size for the nginx-cache volume
364+
size: "256Mi"
365+
## Storage class for the nginx-cache volume. If empty, uses the cluster default.
366+
storageClass: ""
367+
## Access modes for the nginx-cache volume
368+
accessModes:
369+
- "ReadWriteOnce"
370+
347371
## Extra containers for the Ingress Controller pods.
348372
extraContainers: []
349373
# - name: container

0 commit comments

Comments
 (0)
0