diff --git a/build/Dockerfile b/build/Dockerfile index 2ac5b23b83..1f43d2b771 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -376,6 +376,7 @@ RUN --mount=type=bind,target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /et && setcap -v 'cap_net_bind_service=+eip' /usr/sbin/nginx 'cap_net_bind_service=+eip' /usr/sbin/nginx-debug \ && [ -z "${BUILD_OS##*plus*}" ] && PLUS=-plus; cp -a /tmp/internal/configs/version1/nginx$PLUS.ingress.tmpl /tmp/internal/configs/version1/nginx$PLUS.tmpl \ /tmp/internal/configs/version2/nginx$PLUS.virtualserver.tmpl /tmp/internal/configs/version2/nginx$PLUS.transportserver.tmpl / \ + && if [ -z "${BUILD_OS##*plus*}" ]; then mkdir -p /etc/nginx/state_files/; fi \ && chown -R 101:0 /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl \ && chmod -R g=u /etc/nginx /var/cache/nginx /var/lib/nginx /var/log/nginx /*.tmpl \ && rm -f /etc/nginx/conf.d/* diff --git a/charts/nginx-ingress/README.md b/charts/nginx-ingress/README.md index 05f72f11ee..fb4289459e 100644 --- a/charts/nginx-ingress/README.md +++ b/charts/nginx-ingress/README.md @@ -471,6 +471,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont |`controller.readOnlyRootFilesystem` | Configure root filesystem as read-only and add volumes for temporary data. Three major releases after 3.5.x this argument will be moved permanently to the `controller.securityContext` section. | false | |`controller.enableSSLDynamicReload` | Enable lazy loading for SSL Certificates. | true | |`controller.telemetryReporting.enable` | Enable telemetry reporting. | true | +|`controller.enableDynamicWeightChangesReload` | Enable weight changes without reloading the NGINX configuration. May require increasing map_hash_bucket_size, map_hash_max_size, variable_hash_bucket_size, and variable_hash_max_size in the [ConfigMap](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) if there are many two-way splits. Requires `controller.nginxplus` | false | |`rbac.create` | Configures RBAC. | true | |`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | |`prometheus.port` | Configures the port to scrape the metrics. | 9113 | diff --git a/charts/nginx-ingress/templates/_helpers.tpl b/charts/nginx-ingress/templates/_helpers.tpl index 6bd2bd1783..497e1f6cdb 100644 --- a/charts/nginx-ingress/templates/_helpers.tpl +++ b/charts/nginx-ingress/templates/_helpers.tpl @@ -278,6 +278,7 @@ Build the args for the service binary. - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }} - -ssl-dynamic-reload={{ .Values.controller.enableSSLDynamicReload }} - -enable-telemetry-reporting={{ .Values.controller.telemetryReporting.enable}} +- -weight-changes-dynamic-reload={{ .Values.controller.enableWeightChangesDynamicReload}} {{- if .Values.nginxAgent.enable }} - -agent=true - -agent-instance-group={{ default (include "nginx-ingress.controller.fullname" .) .Values.nginxAgent.instanceGroup }} diff --git a/charts/nginx-ingress/values.schema.json b/charts/nginx-ingress/values.schema.json index 3ccd63bead..36ec62f77f 100644 --- a/charts/nginx-ingress/values.schema.json +++ b/charts/nginx-ingress/values.schema.json @@ -1432,6 +1432,14 @@ ] } } + }, + "enableWeightChangesDynamicReload": { + "type": "boolean", + "default": false, + "title": "Enables weight changes without reloading for NGINX Plus", + "examples": [ + false + ] } }, "examples": [ diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index 8b09d8be87..d78eb7705d 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -492,6 +492,11 @@ controller: ## Enable telemetry reporting enable: true + ## Allows weight adjustments without reloading the NGINX Configuration for two-way splits in NGINX Plus. + ## May require increasing map_hash_bucket_size, map_hash_max_size, + ## variable_hash_bucket_size, and variable_hash_max_size in the ConfigMap based on the number of two-way splits. + enableWeightChangesDynamicReload: false + rbac: ## Configures RBAC. create: true diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index 2ad5ce294d..3d02c86b23 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -16,7 +16,8 @@ import ( ) const ( - dynamicSSLReloadParam = "ssl-dynamic-reload" + dynamicSSLReloadParam = "ssl-dynamic-reload" + dynamicWeightChangesParam = "weight-changes-dynamic-reload" ) var ( @@ -206,6 +207,8 @@ var ( enableTelemetryReporting = flag.Bool("enable-telemetry-reporting", true, "Enable gathering and reporting of product related telemetry.") + enableDynamicWeightChangesReload = flag.Bool(dynamicWeightChangesParam, false, "Enable changing weights of split clients without reloading NGINX. Requires -nginx-plus") + startupCheckFn func() error ) @@ -281,6 +284,11 @@ func parseFlags() { glog.Fatal("ingresslink and external-service cannot both be set") } + if *enableDynamicWeightChangesReload && !*nginxPlus { + glog.Warning("weight-changes-dynamic-reload flag support is for NGINX Plus, Dynamic Weight Changes will not be enabled") + *enableDynamicWeightChangesReload = false + } + if *agent && !*appProtect { glog.Fatal("NGINX Agent is used to enable the Security Monitoring dashboard and requires NGINX App Protect to be enabled") } diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 0c03ce266e..95326892b6 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -123,6 +123,7 @@ func main() { SSLRejectHandshake: sslRejectHandshake, EnableCertManager: *enableCertManager, DynamicSSLReload: *enableDynamicSSLReload, + DynamicWeightChangesReload: *enableDynamicWeightChangesReload, StaticSSLPath: nginxManager.GetSecretsDir(), NginxVersion: nginxVersion, } @@ -140,19 +141,20 @@ func main() { plusCollector, syslogListener, latencyCollector := createPlusAndLatencyCollectors(registry, constLabels, kubeClient, plusClient, staticCfgParams.NginxServiceMesh) cnf := configs.NewConfigurator(configs.ConfiguratorParams{ - NginxManager: nginxManager, - StaticCfgParams: staticCfgParams, - Config: cfgParams, - TemplateExecutor: templateExecutor, - TemplateExecutorV2: templateExecutorV2, - LatencyCollector: latencyCollector, - LabelUpdater: plusCollector, - IsPlus: *nginxPlus, - IsWildcardEnabled: isWildcardEnabled, - IsPrometheusEnabled: *enablePrometheusMetrics, - IsLatencyMetricsEnabled: *enableLatencyMetrics, - IsDynamicSSLReloadEnabled: *enableDynamicSSLReload, - NginxVersion: nginxVersion, + NginxManager: nginxManager, + StaticCfgParams: staticCfgParams, + Config: cfgParams, + TemplateExecutor: templateExecutor, + TemplateExecutorV2: templateExecutorV2, + LatencyCollector: latencyCollector, + LabelUpdater: plusCollector, + IsPlus: *nginxPlus, + IsWildcardEnabled: isWildcardEnabled, + IsPrometheusEnabled: *enablePrometheusMetrics, + IsLatencyMetricsEnabled: *enableLatencyMetrics, + IsDynamicSSLReloadEnabled: *enableDynamicSSLReload, + IsDynamicWeightChangesReloadEnabled: *enableDynamicWeightChangesReload, + NginxVersion: nginxVersion, }) controllerNamespace := os.Getenv("POD_NAMESPACE") @@ -211,6 +213,7 @@ func main() { WatchNamespaceLabel: *watchNamespaceLabel, EnableTelemetryReporting: *enableTelemetryReporting, NICVersion: version, + DynamicWeightChangesReload: *enableDynamicWeightChangesReload, } lbc := k8s.NewLoadBalancerController(lbcInput) diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index 7a0726d9f5..30bacf082f 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -536,6 +536,20 @@ The default value is `true`. +### -weight-changes-dynamic-reload + +Enables the ability to change the weight distribution of two-way split clients without reloading NGINX. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +Using this feature may require increasing `map_hash_bucket_size`, `map_hash_max_size`, `variable_hash_bucket_size`, and `variable_hash_max_size` in the ConfigMap based on the number of two-way splits. + +The default value is `false`. + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will ignore the flag. + + + ### -enable-telemetry-reporting Enable gathering and reporting of software telemetry. diff --git a/go.mod b/go.mod index cb6d0a412a..193bf3f1aa 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang/glog v1.1.2 github.com/google/go-cmp v0.6.0 + github.com/jinzhu/copier v0.4.0 github.com/kr/pretty v0.3.1 github.com/nginxinc/nginx-plus-go-client v1.2.0 github.com/nginxinc/nginx-prometheus-exporter v1.1.0 diff --git a/go.sum b/go.sum index 03547c5347..46d67cd151 100644 --- a/go.sum +++ b/go.sum @@ -149,6 +149,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index 1c06c6c484..73b61cab1f 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -150,6 +150,7 @@ type StaticConfigParams struct { EnableCertManager bool DynamicSSLReload bool StaticSSLPath string + DynamicWeightChangesReload bool NginxVersion nginx.Version } diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 7c41dd0ea7..35c3b395f4 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -84,6 +84,13 @@ type ExtendedResources struct { TransportServerExes []*TransportServerEx } +// WeightUpdate holds the information about the weight updates for weight changes without reloading. +type WeightUpdate struct { + Zone string + Key string + Value string +} + type tlsPassthroughPair struct { Host string UnixSocket string @@ -133,19 +140,20 @@ type Configurator struct { // ConfiguratorParams is a collection of parameters used for the // NewConfigurator() function type ConfiguratorParams struct { - NginxManager nginx.Manager - StaticCfgParams *StaticConfigParams - Config *ConfigParams - TemplateExecutor *version1.TemplateExecutor - TemplateExecutorV2 *version2.TemplateExecutor - LabelUpdater collector.LabelUpdater - LatencyCollector latCollector.LatencyCollector - IsPlus bool - IsPrometheusEnabled bool - IsWildcardEnabled bool - IsLatencyMetricsEnabled bool - IsDynamicSSLReloadEnabled bool - NginxVersion nginx.Version + NginxManager nginx.Manager + StaticCfgParams *StaticConfigParams + Config *ConfigParams + TemplateExecutor *version1.TemplateExecutor + TemplateExecutorV2 *version2.TemplateExecutor + LabelUpdater collector.LabelUpdater + LatencyCollector latCollector.LatencyCollector + IsPlus bool + IsPrometheusEnabled bool + IsWildcardEnabled bool + IsLatencyMetricsEnabled bool + IsDynamicSSLReloadEnabled bool + IsDynamicWeightChangesReloadEnabled bool + NginxVersion nginx.Version } // NewConfigurator creates a new Configurator. @@ -555,15 +563,23 @@ func (cnf *Configurator) deleteVirtualServerMetricsLabels(key string) { // AddOrUpdateVirtualServer adds or updates NGINX configuration for the VirtualServer resource. func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) { - _, warnings, err := cnf.addOrUpdateVirtualServer(virtualServerEx) + _, warnings, weightUpdates, err := cnf.addOrUpdateVirtualServer(virtualServerEx) if err != nil { return warnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err) } + if len(weightUpdates) > 0 { + cnf.EnableReloads() + } + if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { return warnings, fmt.Errorf("error reloading NGINX for VirtualServer %v/%v: %w", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name, err) } + for _, weightUpdate := range weightUpdates { + cnf.nginxManager.UpsertSplitClientsKeyVal(weightUpdate.Zone, weightUpdate.Key, weightUpdate.Value) + } + return warnings, nil } @@ -571,7 +587,8 @@ func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) erro return cnf.nginxManager.CreateOpenTracingTracerConfig(content) } -func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, error) { +func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (bool, Warnings, []WeightUpdate, error) { + var weightUpdates []WeightUpdate apResources := cnf.updateApResourcesForVs(virtualServerEx) dosResources := map[string]*appProtectDosResource{} for k, v := range virtualServerEx.DosProtectedEx { @@ -588,7 +605,7 @@ func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServer vsCfg, warnings := vsc.GenerateVirtualServerConfig(virtualServerEx, apResources, dosResources) content, err := cnf.templateExecutorV2.ExecuteVirtualServerTemplate(&vsCfg) if err != nil { - return false, warnings, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) + return false, warnings, weightUpdates, fmt.Errorf("error generating VirtualServer config: %v: %w", name, err) } changed := cnf.nginxManager.CreateConfig(name, content) @@ -597,25 +614,42 @@ func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServer if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.updateVirtualServerMetricsLabels(virtualServerEx, vsCfg.Upstreams) } - return changed, warnings, nil + + if cnf.staticCfgParams.DynamicWeightChangesReload && len(vsCfg.TwoWaySplitClients) > 0 { + for _, splitClient := range vsCfg.TwoWaySplitClients { + if len(splitClient.Weights) != 2 { + continue + } + variableNamer := *NewVSVariableNamer(virtualServerEx.VirtualServer) + value := variableNamer.GetNameOfKeyOfMapForWeights(splitClient.SplitClientsIndex, splitClient.Weights[0], splitClient.Weights[1]) + weightUpdates = append(weightUpdates, WeightUpdate{Zone: splitClient.ZoneName, Key: splitClient.Key, Value: value}) + } + } + return changed, warnings, weightUpdates, nil } // AddOrUpdateVirtualServers adds or updates NGINX configuration for multiple VirtualServer resources. func (cnf *Configurator) AddOrUpdateVirtualServers(virtualServerExes []*VirtualServerEx) (Warnings, error) { allWarnings := newWarnings() + allWeightUpdates := []WeightUpdate{} for _, vsEx := range virtualServerExes { - _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, weightUpdates, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } allWarnings.Add(warnings) + allWeightUpdates = append(allWeightUpdates, weightUpdates...) } if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { return allWarnings, fmt.Errorf("error when reloading NGINX when updating Policy: %w", err) } + for _, weightUpdate := range allWeightUpdates { + cnf.nginxManager.UpsertSplitClientsKeyVal(weightUpdate.Zone, weightUpdate.Key, weightUpdate.Value) + } + return allWarnings, nil } @@ -794,6 +828,7 @@ func (cnf *Configurator) addOrUpdateHtpasswdSecret(secret *api_v1.Secret) string // AddOrUpdateResources adds or updates configuration for resources. func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloadIfUnchanged bool) (Warnings, error) { allWarnings := newWarnings() + allWeightUpdates := []WeightUpdate{} configsChanged := false updateResource := func(updateFunc func() (bool, Warnings, error), namespace, name string) error { @@ -808,6 +843,20 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloa return nil } + updateVSResource := func(updateFunc func() (bool, Warnings, []WeightUpdate, error), namespace, name string) error { + changed, warnings, weightUpdates, err := updateFunc() + if err != nil { + return fmt.Errorf("error adding or updating resource %v/%v: %w", namespace, name, err) + } + allWarnings.Add(warnings) + allWeightUpdates = append(allWeightUpdates, weightUpdates...) + + if changed { + configsChanged = true + } + return nil + } + for _, ingEx := range resources.IngressExes { err := updateResource(func() (bool, Warnings, error) { return cnf.addOrUpdateIngress(ingEx) @@ -827,7 +876,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources, reloa } for _, vsEx := range resources.VirtualServerExes { - err := updateResource(func() (bool, Warnings, error) { + err := updateVSResource(func() (bool, Warnings, []WeightUpdate, error) { return cnf.addOrUpdateVirtualServer(vsEx) }, vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name) if err != nil { @@ -925,6 +974,10 @@ func (cnf *Configurator) DeleteVirtualServer(key string, skipReload bool) error name := getFileNameForVirtualServerFromKey(key) cnf.nginxManager.DeleteConfig(name) + if cnf.isPlus { + cnf.nginxManager.DeleteKeyValStateFiles(name) + } + delete(cnf.virtualServers, name) if (cnf.isPlus && cnf.isPrometheusEnabled) || cnf.isLatencyMetricsEnabled { cnf.deleteVirtualServerMetricsLabels(key) @@ -1045,7 +1098,7 @@ func (cnf *Configurator) UpdateEndpointsForVirtualServers(virtualServerExes []*V for _, vs := range virtualServerExes { // It is safe to ignore warnings here as no new warnings should appear when updating Endpoints for VirtualServers - _, _, err := cnf.addOrUpdateVirtualServer(vs) + _, _, _, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } @@ -1226,6 +1279,7 @@ func (cnf *Configurator) updateStreamServersInPlus(upstream string, servers []st func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources ExtendedResources) (Warnings, error) { cnf.cfgParams = cfgParams allWarnings := newWarnings() + allWeightUpdates := []WeightUpdate{} if cnf.cfgParams.MainServerSSLDHParamFileContent != nil { fileName, err := cnf.nginxManager.CreateDHParam(*cnf.cfgParams.MainServerSSLDHParamFileContent) @@ -1278,11 +1332,12 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende allWarnings.Add(warnings) } for _, vsEx := range resources.VirtualServerExes { - _, warnings, err := cnf.addOrUpdateVirtualServer(vsEx) + _, warnings, weightUpdates, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { return allWarnings, err } allWarnings.Add(warnings) + allWeightUpdates = append(allWeightUpdates, weightUpdates...) } for _, tsEx := range resources.TransportServerExes { @@ -1304,6 +1359,10 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende return allWarnings, fmt.Errorf("error when updating config from ConfigMap: %w", err) } + for _, weightUpdate := range allWeightUpdates { + cnf.nginxManager.UpsertSplitClientsKeyVal(weightUpdate.Zone, weightUpdate.Key, weightUpdate.Value) + } + return allWarnings, nil } @@ -1321,11 +1380,13 @@ func (cnf *Configurator) ReloadForBatchUpdates(batchReloadsEnabled bool) error { // UpdateVirtualServers updates VirtualServers. func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, deletedKeys []string) []error { var errList []error + var allWeightUpdates []WeightUpdate for _, vsEx := range updatedVSExes { - _, _, err := cnf.addOrUpdateVirtualServer(vsEx) + _, _, weightUpdates, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { errList = append(errList, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err)) } + allWeightUpdates = append(allWeightUpdates, weightUpdates...) } for _, key := range deletedKeys { @@ -1339,6 +1400,10 @@ func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, errList = append(errList, fmt.Errorf("error when updating VirtualServer: %w", err)) } + for _, weightUpdate := range allWeightUpdates { + cnf.nginxManager.UpsertSplitClientsKeyVal(weightUpdate.Zone, weightUpdate.Key, weightUpdate.Value) + } + return errList } @@ -1640,6 +1705,7 @@ func (cnf *Configurator) AddOrUpdateResourcesThatUseDosProtected(ingExes []*Ingr func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*IngressEx, mergeableIngresses []*MergeableIngresses, vsExes []*VirtualServerEx) (Warnings, error) { allWarnings := newWarnings() + allWeightUpdates := []WeightUpdate{} for _, ingEx := range ingExes { _, warnings, err := cnf.addOrUpdateIngress(ingEx) @@ -1658,13 +1724,18 @@ func (cnf *Configurator) addOrUpdateIngressesAndVirtualServers(ingExes []*Ingres } for _, vs := range vsExes { - _, warnings, err := cnf.addOrUpdateVirtualServer(vs) + _, warnings, weightUpdates, err := cnf.addOrUpdateVirtualServer(vs) if err != nil { return allWarnings, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vs.VirtualServer.Namespace, vs.VirtualServer.Name, err) } + allWeightUpdates = append(allWeightUpdates, weightUpdates...) allWarnings.Add(warnings) } + for _, weightUpdate := range allWeightUpdates { + cnf.nginxManager.UpsertSplitClientsKeyVal(weightUpdate.Zone, weightUpdate.Key, weightUpdate.Value) + } + return allWarnings, nil } @@ -1775,3 +1846,8 @@ func (cnf *Configurator) DeleteSecret(key string) { func (cnf *Configurator) DynamicSSLReloadEnabled() bool { return cnf.isDynamicSSLReloadEnabled } + +// UpsertSplitClientsKeyVal upserts a key-value pair in a keyzal zone for weight changes without reloads. +func (cnf *Configurator) UpsertSplitClientsKeyVal(zoneName, key, value string) { + cnf.nginxManager.UpsertSplitClientsKeyVal(zoneName, key, value) +} diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 0cf4575b21..e525a533c6 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -38,6 +38,8 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; + map_hash_max_size {{.MapHashMaxSize}}; + map_hash_bucket_size {{.MapHashBucketSize}}; {{- if .HTTPSnippets}} {{range $value := .HTTPSnippets}} diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 4c45f186d2..a3e34a94f9 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -27,6 +27,8 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; + map_hash_max_size {{.MapHashMaxSize}}; + map_hash_bucket_size {{.MapHashBucketSize}}; {{- if .HTTPSnippets}} {{range $value := .HTTPSnippets}} diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index 228582ff5d..6d6eae07cc 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -13,6 +13,9 @@ type UpstreamLabels struct { // VirtualServerConfig holds NGINX configuration for a VirtualServer. type VirtualServerConfig struct { HTTPSnippets []string + TwoWaySplitClients []TwoWaySplitClients + KeyValZones []KeyValZone + KeyVals []KeyVal LimitReqZones []LimitReqZone Maps []Map Server Server @@ -387,3 +390,32 @@ type BasicAuth struct { Secret string Realm string } + +// KeyValZone defines a keyval zone. +type KeyValZone struct { + Name string + Size string + State string +} + +// KeyVal defines a keyval. +type KeyVal struct { + Key string + Variable string + ZoneName string +} + +// TwoWaySplitClients defines split clients for two way split +type TwoWaySplitClients struct { + Key string + Variable string + ZoneName string + Weights []int + SplitClientsIndex int +} + +// Variable defines an nginx variable. +type Variable struct { + Name string + Value string +} diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index cf84d117b2..61c3c2ba3b 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -32,6 +32,14 @@ upstream {{ $u.Name }} { } {{ end }} +{{- range $kvz := .KeyValZones }} +keyval_zone zone={{ $kvz.Name }}:{{ $kvz.Size}} state={{ $kvz.State }}; +{{- end }} + +{{- range $kv := .KeyVals }} +keyval {{ $kv.Key}} {{ $kv.Variable}} zone={{ $kv.ZoneName }}; +{{- end }} + {{- range $sc := .SplitClients }} split_clients {{ $sc.Source }} {{ $sc.Variable }} { {{- range $d := $sc.Distributions }} diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index ff6ee6f35d..5dfff6a3b8 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -21,13 +21,16 @@ import ( ) const ( - nginx502Server = "unix:/var/lib/nginx/nginx-502-server.sock" - internalLocationPrefix = "internal_location_" - nginx418Server = "unix:/var/lib/nginx/nginx-418-server.sock" - specContext = "spec" - routeContext = "route" - subRouteContext = "subroute" - defaultLogOutput = "syslog:server=localhost:514" + nginx502Server = "unix:/var/lib/nginx/nginx-502-server.sock" + internalLocationPrefix = "internal_location_" + nginx418Server = "unix:/var/lib/nginx/nginx-418-server.sock" + specContext = "spec" + routeContext = "route" + subRouteContext = "subroute" + keyvalZoneBasePath = "/etc/nginx/state_files" + splitClientsKeyValZoneSize = "100k" + splitClientAmountWhenWeightChangesDynamicReload = 101 + defaultLogOutput = "syslog:server=localhost:514" ) var grpcConflictingErrors = map[int]bool{ @@ -178,22 +181,56 @@ func (namer *upstreamNamer) GetNameForUpstream(upstream string) string { return fmt.Sprintf("%s_%s", namer.prefix, upstream) } -type variableNamer struct { +// VariableNamer is a namer which generates unique variable names for a VirtualServer. +type VariableNamer struct { safeNsName string } -func newVariableNamer(virtualServer *conf_v1.VirtualServer) *variableNamer { +// NewVSVariableNamer creates a new namer for a VirtualServer. +func NewVSVariableNamer(virtualServer *conf_v1.VirtualServer) *VariableNamer { safeNsName := strings.ReplaceAll(fmt.Sprintf("%s_%s", virtualServer.Namespace, virtualServer.Name), "-", "_") - return &variableNamer{ + return &VariableNamer{ safeNsName: safeNsName, } } -func (namer *variableNamer) GetNameForSplitClientVariable(index int) string { +// GetNameOfKeyvalZoneForSplitClientIndex returns a unique name for a keyval zone for split clients. +func (namer *VariableNamer) GetNameOfKeyvalZoneForSplitClientIndex(index int) string { + return fmt.Sprintf("vs_%s_keyval_zone_split_clients_%d", namer.safeNsName, index) +} + +// GetNameOfKeyvalForSplitClientIndex returns a unique name for a keyval for split clients. +func (namer *VariableNamer) GetNameOfKeyvalForSplitClientIndex(index int) string { + return fmt.Sprintf("$vs_%s_keyval_split_clients_%d", namer.safeNsName, index) +} + +// GetNameOfKeyvalKeyForSplitClientIndex returns a unique name for a keyval key for split clients. +func (namer *VariableNamer) GetNameOfKeyvalKeyForSplitClientIndex(index int) string { + return fmt.Sprintf("\"vs_%s_keyval_key_split_clients_%d\"", namer.safeNsName, index) +} + +// GetNameOfMapForSplitClientIndex returns a unique name for a map for split clients. +func (namer *VariableNamer) GetNameOfMapForSplitClientIndex(index int) string { + return fmt.Sprintf("$vs_%s_map_split_clients_%d", namer.safeNsName, index) +} + +// GetNameOfKeyOfMapForWeights returns a unique name for a key of a map for split clients. +func (namer *VariableNamer) GetNameOfKeyOfMapForWeights(index int, i int, j int) string { + return fmt.Sprintf("\"vs_%s_split_clients_%d_%d_%d\"", namer.safeNsName, index, i, j) +} + +// GetNameOfSplitClientsForWeights gets the name of the split clients for a particular combination of weights and scIndex. +func (namer *VariableNamer) GetNameOfSplitClientsForWeights(index int, i int, j int) string { + return fmt.Sprintf("$vs_%s_split_clients_%d_%d_%d", namer.safeNsName, index, i, j) +} + +// GetNameForSplitClientVariable gets the name of a split client variable for a particular scIndex. +func (namer *VariableNamer) GetNameForSplitClientVariable(index int) string { return fmt.Sprintf("$vs_%s_splits_%d", namer.safeNsName, index) } -func (namer *variableNamer) GetNameForVariableForMatchesRouteMap( +// GetNameForVariableForMatchesRouteMap gets the name of a matches route map +func (namer *VariableNamer) GetNameForVariableForMatchesRouteMap( matchesIndex int, matchIndex int, conditionIndex int, @@ -201,7 +238,8 @@ func (namer *variableNamer) GetNameForVariableForMatchesRouteMap( return fmt.Sprintf("$vs_%s_matches_%d_match_%d_cond_%d", namer.safeNsName, matchesIndex, matchIndex, conditionIndex) } -func (namer *variableNamer) GetNameForVariableForMatchesRouteMainMap(matchesIndex int) string { +// GetNameForVariableForMatchesRouteMainMap gets the name of a matches route main map +func (namer *VariableNamer) GetNameForVariableForMatchesRouteMainMap(matchesIndex int) string { return fmt.Sprintf("$vs_%s_matches_%d", namer.safeNsName, matchesIndex) } @@ -230,20 +268,21 @@ func newHealthCheckWithDefaults(upstream conf_v1.Upstream, upstreamName string, // VirtualServerConfigurator generates a VirtualServer configuration type virtualServerConfigurator struct { - cfgParams *ConfigParams - isPlus bool - isWildcardEnabled bool - isResolverConfigured bool - isTLSPassthrough bool - enableSnippets bool - warnings Warnings - spiffeCerts bool - enableInternalRoutes bool - oidcPolCfg *oidcPolicyCfg - isIPV6Disabled bool - DynamicSSLReloadEnabled bool - StaticSSLPath string - bundleValidator bundleValidator + cfgParams *ConfigParams + isPlus bool + isWildcardEnabled bool + isResolverConfigured bool + isTLSPassthrough bool + enableSnippets bool + warnings Warnings + spiffeCerts bool + enableInternalRoutes bool + oidcPolCfg *oidcPolicyCfg + isIPV6Disabled bool + DynamicSSLReloadEnabled bool + StaticSSLPath string + DynamicWeightChangesReload bool + bundleValidator bundleValidator } type oidcPolicyCfg struct { @@ -278,20 +317,21 @@ func newVirtualServerConfigurator( bundleValidator = newInternalBundleValidator(appProtectBundleFolder) } return &virtualServerConfigurator{ - cfgParams: cfgParams, - isPlus: isPlus, - isWildcardEnabled: isWildcardEnabled, - isResolverConfigured: isResolverConfigured, - isTLSPassthrough: staticParams.TLSPassthrough, - enableSnippets: staticParams.EnableSnippets, - warnings: make(map[runtime.Object][]string), - spiffeCerts: staticParams.NginxServiceMesh, - enableInternalRoutes: staticParams.EnableInternalRoutes, - oidcPolCfg: &oidcPolicyCfg{}, - isIPV6Disabled: staticParams.DisableIPV6, - DynamicSSLReloadEnabled: staticParams.DynamicSSLReload, - StaticSSLPath: staticParams.StaticSSLPath, - bundleValidator: bundleValidator, + cfgParams: cfgParams, + isPlus: isPlus, + isWildcardEnabled: isWildcardEnabled, + isResolverConfigured: isResolverConfigured, + isTLSPassthrough: staticParams.TLSPassthrough, + enableSnippets: staticParams.EnableSnippets, + warnings: make(map[runtime.Object][]string), + spiffeCerts: staticParams.NginxServiceMesh, + enableInternalRoutes: staticParams.EnableInternalRoutes, + oidcPolCfg: &oidcPolicyCfg{}, + isIPV6Disabled: staticParams.DisableIPV6, + DynamicSSLReloadEnabled: staticParams.DynamicSSLReload, + StaticSSLPath: staticParams.StaticSSLPath, + DynamicWeightChangesReload: staticParams.DynamicWeightChangesReload, + bundleValidator: bundleValidator, } } @@ -469,6 +509,9 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( var splitClients []version2.SplitClient var maps []version2.Map var errorPageLocations []version2.ErrorPageLocation + var keyValZones []version2.KeyValZone + var keyVals []version2.KeyVal + var twoWaySplitClients []version2.TwoWaySplitClients vsrErrorPagesFromVs := make(map[string][]conf_v1.ErrorPage) vsrErrorPagesRouteIndex := make(map[string]int) vsrLocationSnippetsFromVs := make(map[string]string) @@ -476,7 +519,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( isVSR := false matchesRoutes := 0 - variableNamer := newVariableNamer(vsEx.VirtualServer) + VariableNamer := NewVSVariableNamer(vsEx.VirtualServer) // generates config for VirtualServer routes for _, r := range vsEx.VirtualServer.Spec.Routes { @@ -545,7 +588,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( r, virtualServerUpstreamNamer, crUpstreams, - variableNamer, + VariableNamer, matchesRoutes, len(splitClients), vsc.cfgParams, @@ -556,6 +599,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( isVSR, "", "", vsc.warnings, + vsc.DynamicWeightChangesReload, ) addPoliciesCfgToLocations(routePoliciesCfg, cfg.Locations) addDosConfigToLocations(dosRouteCfg, cfg.Locations) @@ -565,16 +609,23 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) splitClients = append(splitClients, cfg.SplitClients...) + keyValZones = append(keyValZones, cfg.KeyValZones...) + keyVals = append(keyVals, cfg.KeyVals...) + twoWaySplitClients = append(twoWaySplitClients, cfg.TwoWaySplitClients...) matchesRoutes++ } else if len(r.Splits) > 0 { - cfg := generateDefaultSplitsConfig(r, virtualServerUpstreamNamer, crUpstreams, variableNamer, len(splitClients), - vsc.cfgParams, errorPages, r.Path, vsLocSnippets, vsc.enableSnippets, len(returnLocations), isVSR, "", "", vsc.warnings) + cfg := generateDefaultSplitsConfig(r, virtualServerUpstreamNamer, crUpstreams, VariableNamer, len(splitClients), + vsc.cfgParams, errorPages, r.Path, vsLocSnippets, vsc.enableSnippets, len(returnLocations), isVSR, "", "", vsc.warnings, vsc.DynamicWeightChangesReload) addPoliciesCfgToLocations(routePoliciesCfg, cfg.Locations) addDosConfigToLocations(dosRouteCfg, cfg.Locations) splitClients = append(splitClients, cfg.SplitClients...) locations = append(locations, cfg.Locations...) internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) + maps = append(maps, cfg.Maps...) + keyValZones = append(keyValZones, cfg.KeyValZones...) + keyVals = append(keyVals, cfg.KeyVals...) + twoWaySplitClients = append(twoWaySplitClients, cfg.TwoWaySplitClients...) } else { upstreamName := virtualServerUpstreamNamer.GetNameForUpstreamFromAction(r.Action) upstream := crUpstreams[upstreamName] @@ -667,7 +718,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( r, upstreamNamer, crUpstreams, - variableNamer, + VariableNamer, matchesRoutes, len(splitClients), vsc.cfgParams, @@ -679,6 +730,7 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( vsr.Name, vsr.Namespace, vsc.warnings, + vsc.DynamicWeightChangesReload, ) addPoliciesCfgToLocations(routePoliciesCfg, cfg.Locations) addDosConfigToLocations(dosRouteCfg, cfg.Locations) @@ -688,10 +740,13 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) splitClients = append(splitClients, cfg.SplitClients...) + keyValZones = append(keyValZones, cfg.KeyValZones...) + keyVals = append(keyVals, cfg.KeyVals...) + twoWaySplitClients = append(twoWaySplitClients, cfg.TwoWaySplitClients...) matchesRoutes++ } else if len(r.Splits) > 0 { - cfg := generateDefaultSplitsConfig(r, upstreamNamer, crUpstreams, variableNamer, len(splitClients), vsc.cfgParams, - errorPages, r.Path, locSnippets, vsc.enableSnippets, len(returnLocations), isVSR, vsr.Name, vsr.Namespace, vsc.warnings) + cfg := generateDefaultSplitsConfig(r, upstreamNamer, crUpstreams, VariableNamer, len(splitClients), vsc.cfgParams, + errorPages, r.Path, locSnippets, vsc.enableSnippets, len(returnLocations), isVSR, vsr.Name, vsr.Namespace, vsc.warnings, vsc.DynamicWeightChangesReload) addPoliciesCfgToLocations(routePoliciesCfg, cfg.Locations) addDosConfigToLocations(dosRouteCfg, cfg.Locations) @@ -699,6 +754,10 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( locations = append(locations, cfg.Locations...) internalRedirectLocations = append(internalRedirectLocations, cfg.InternalRedirectLocation) returnLocations = append(returnLocations, cfg.ReturnLocations...) + keyValZones = append(keyValZones, cfg.KeyValZones...) + keyVals = append(keyVals, cfg.KeyVals...) + twoWaySplitClients = append(twoWaySplitClients, cfg.TwoWaySplitClients...) + maps = append(maps, cfg.Maps...) } else { upstreamName := upstreamNamer.GetNameForUpstreamFromAction(r.Action) upstream := crUpstreams[upstreamName] @@ -778,6 +837,9 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( SpiffeClientCerts: vsc.spiffeCerts && !enabledInternalRoutes, DynamicSSLReloadEnabled: vsc.DynamicSSLReloadEnabled, StaticSSLPath: vsc.StaticSSLPath, + KeyValZones: keyValZones, + KeyVals: keyVals, + TwoWaySplitClients: twoWaySplitClients, } return vsCfg, vsc.warnings @@ -2054,13 +2116,16 @@ type routingCfg struct { Locations []version2.Location InternalRedirectLocation version2.InternalRedirectLocation ReturnLocations []version2.ReturnLocation + KeyValZones []version2.KeyValZone + KeyVals []version2.KeyVal + TwoWaySplitClients []version2.TwoWaySplitClients } func generateSplits( splits []conf_v1.Split, upstreamNamer *upstreamNamer, crUpstreams map[string]conf_v1.Upstream, - variableNamer *variableNamer, + VariableNamer *VariableNamer, scIndex int, cfgParams *ConfigParams, errorPages errorPageDetails, @@ -2072,8 +2137,14 @@ func generateSplits( vsrName string, vsrNamespace string, vscWarnings Warnings, -) (version2.SplitClient, []version2.Location, []version2.ReturnLocation) { + WeightChangesDynamicReload bool, +) ([]version2.SplitClient, []version2.Location, []version2.ReturnLocation, []version2.Map, []version2.KeyValZone, []version2.KeyVal, []version2.TwoWaySplitClients) { var distributions []version2.Distribution + var splitClients []version2.SplitClient + var maps []version2.Map + var keyValZones []version2.KeyValZone + var keyVals []version2.KeyVal + var twoWaySplitClients []version2.TwoWaySplitClients for i, s := range splits { if s.Weight == 0 { @@ -2086,10 +2157,38 @@ func generateSplits( distributions = append(distributions, d) } - splitClient := version2.SplitClient{ - Source: "$request_id", - Variable: variableNamer.GetNameForSplitClientVariable(scIndex), - Distributions: distributions, + if WeightChangesDynamicReload && len(splits) == 2 { + scs, weightMap := generateSplitsForWeightChangesDynamicReload(splits, scIndex, VariableNamer) + kvZoneName := VariableNamer.GetNameOfKeyvalZoneForSplitClientIndex(scIndex) + kvz := version2.KeyValZone{ + Name: kvZoneName, + Size: splitClientsKeyValZoneSize, + State: fmt.Sprintf("%s/%s.json", keyvalZoneBasePath, kvZoneName), + } + kv := version2.KeyVal{ + Key: VariableNamer.GetNameOfKeyvalKeyForSplitClientIndex(scIndex), + Variable: VariableNamer.GetNameOfKeyvalForSplitClientIndex(scIndex), + ZoneName: kvZoneName, + } + scWithWeights := version2.TwoWaySplitClients{ + Key: VariableNamer.GetNameOfKeyvalKeyForSplitClientIndex(scIndex), + Variable: VariableNamer.GetNameOfKeyvalForSplitClientIndex(scIndex), + ZoneName: kvZoneName, + Weights: []int{splits[0].Weight, splits[1].Weight}, + SplitClientsIndex: scIndex, + } + splitClients = append(splitClients, scs...) + maps = append(maps, weightMap) + keyValZones = append(keyValZones, kvz) + keyVals = append(keyVals, kv) + twoWaySplitClients = append(twoWaySplitClients, scWithWeights) + } else { + splitClient := version2.SplitClient{ + Source: "$request_id", + Variable: VariableNamer.GetNameForSplitClientVariable(scIndex), + Distributions: distributions, + } + splitClients = append(splitClients, splitClient) } var locations []version2.Location @@ -2109,14 +2208,14 @@ func generateSplits( } } - return splitClient, locations, returnLocations + return splitClients, locations, returnLocations, maps, keyValZones, keyVals, twoWaySplitClients } func generateDefaultSplitsConfig( route conf_v1.Route, upstreamNamer *upstreamNamer, crUpstreams map[string]conf_v1.Upstream, - variableNamer *variableNamer, + VariableNamer *VariableNamer, scIndex int, cfgParams *ConfigParams, errorPages errorPageDetails, @@ -2128,39 +2227,105 @@ func generateDefaultSplitsConfig( vsrName string, vsrNamespace string, vscWarnings Warnings, + weightChangesDynamicReload bool, ) routingCfg { - sc, locs, returnLocs := generateSplits(route.Splits, upstreamNamer, crUpstreams, variableNamer, scIndex, cfgParams, - errorPages, originalPath, locSnippets, enableSnippets, retLocIndex, isVSR, vsrName, vsrNamespace, vscWarnings) - - splitClientVarName := variableNamer.GetNameForSplitClientVariable(scIndex) + scs, locs, returnLocs, maps, keyValZones, keyVals, twoWaySplitClients := generateSplits(route.Splits, upstreamNamer, crUpstreams, VariableNamer, scIndex, cfgParams, errorPages, originalPath, locSnippets, enableSnippets, retLocIndex, isVSR, vsrName, vsrNamespace, vscWarnings, weightChangesDynamicReload) - irl := version2.InternalRedirectLocation{ - Path: route.Path, - Destination: splitClientVarName, + var irl version2.InternalRedirectLocation + if weightChangesDynamicReload && len(route.Splits) == 2 { + irl = version2.InternalRedirectLocation{ + Path: route.Path, + Destination: VariableNamer.GetNameOfMapForSplitClientIndex(scIndex), + } + } else { + irl = version2.InternalRedirectLocation{ + Path: route.Path, + Destination: VariableNamer.GetNameForSplitClientVariable(scIndex), + } } return routingCfg{ - SplitClients: []version2.SplitClient{sc}, + SplitClients: scs, Locations: locs, InternalRedirectLocation: irl, ReturnLocations: returnLocs, + Maps: maps, + KeyValZones: keyValZones, + KeyVals: keyVals, + TwoWaySplitClients: twoWaySplitClients, } } +func generateSplitsForWeightChangesDynamicReload(splits []conf_v1.Split, scIndex int, VariableNamer *VariableNamer) ([]version2.SplitClient, version2.Map) { + var splitClients []version2.SplitClient + var mapParameters []version2.Parameter + for i := 0; i <= 100; i++ { + j := 100 - i + var split version2.SplitClient + var distributions []version2.Distribution + if i > 0 { + distribution := version2.Distribution{ + Weight: fmt.Sprintf("%d%%", i), + Value: fmt.Sprintf("/%vsplits_%d_split_%d", internalLocationPrefix, scIndex, 0), + } + distributions = append(distributions, distribution) + + } + if j > 0 { + distribution := version2.Distribution{ + Weight: fmt.Sprintf("%d%%", j), + Value: fmt.Sprintf("/%vsplits_%d_split_%d", internalLocationPrefix, scIndex, 1), + } + distributions = append(distributions, distribution) + } + split = version2.SplitClient{ + Source: "$request_id", + Variable: VariableNamer.GetNameOfSplitClientsForWeights(scIndex, i, j), + Distributions: distributions, + } + splitClients = append(splitClients, split) + mapParameters = append(mapParameters, version2.Parameter{ + Value: VariableNamer.GetNameOfKeyOfMapForWeights(scIndex, i, j), + Result: VariableNamer.GetNameOfSplitClientsForWeights(scIndex, i, j), + }) + + } + + var mapDefault version2.Parameter + var result string + if splits[0].Weight < splits[1].Weight { + result = VariableNamer.GetNameOfSplitClientsForWeights(scIndex, 0, 100) + } else { + result = VariableNamer.GetNameOfSplitClientsForWeights(scIndex, 100, 0) + } + mapDefault = version2.Parameter{Value: "default", Result: result} + + mapParameters = append(mapParameters, mapDefault) + + weightsToSplits := version2.Map{ + Source: VariableNamer.GetNameOfKeyvalForSplitClientIndex(scIndex), + Variable: VariableNamer.GetNameOfMapForSplitClientIndex(scIndex), + Parameters: mapParameters, + } + + return splitClients, weightsToSplits +} + func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, crUpstreams map[string]conf_v1.Upstream, - variableNamer *variableNamer, index int, scIndex int, cfgParams *ConfigParams, errorPages errorPageDetails, - locSnippets string, enableSnippets bool, retLocIndex int, isVSR bool, vsrName string, vsrNamespace string, vscWarnings Warnings, + VariableNamer *VariableNamer, index int, scIndex int, cfgParams *ConfigParams, errorPages errorPageDetails, + locSnippets string, enableSnippets bool, retLocIndex int, isVSR bool, vsrName string, vsrNamespace string, vscWarnings Warnings, weightChangesDynamicReload bool, ) routingCfg { // Generate maps var maps []version2.Map + var twoWaySplitClients []version2.TwoWaySplitClients for i, m := range route.Matches { for j, c := range m.Conditions { source := getNameForSourceForMatchesRouteMapFromCondition(c) - variable := variableNamer.GetNameForVariableForMatchesRouteMap(index, i, j) + variable := VariableNamer.GetNameForVariableForMatchesRouteMap(index, i, j) successfulResult := "1" if j < len(m.Conditions)-1 { - successfulResult = variableNamer.GetNameForVariableForMatchesRouteMap(index, i, j+1) + successfulResult = VariableNamer.GetNameForVariableForMatchesRouteMap(index, i, j+1) } params := generateParametersForMatchesRouteMap(c.Value, successfulResult) @@ -2180,13 +2345,18 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr source := "" var params []version2.Parameter for i, m := range route.Matches { - source += variableNamer.GetNameForVariableForMatchesRouteMap(index, i, 0) + source += VariableNamer.GetNameForVariableForMatchesRouteMap(index, i, 0) v := fmt.Sprintf("~^%s1", strings.Repeat("0", i)) r := fmt.Sprintf("/%vmatches_%d_match_%d", internalLocationPrefix, index, i) if len(m.Splits) > 0 { - r = variableNamer.GetNameForSplitClientVariable(scIndex + scLocalIndex) - scLocalIndex++ + if weightChangesDynamicReload && len(m.Splits) == 2 { + r = VariableNamer.GetNameOfMapForSplitClientIndex(scIndex + scLocalIndex) + scLocalIndex += splitClientAmountWhenWeightChangesDynamicReload + } else { + r = VariableNamer.GetNameForSplitClientVariable(scIndex + scLocalIndex) + scLocalIndex++ + } } p := version2.Parameter{ @@ -2198,7 +2368,11 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr defaultResult := fmt.Sprintf("/%vmatches_%d_default", internalLocationPrefix, index) if len(route.Splits) > 0 { - defaultResult = variableNamer.GetNameForSplitClientVariable(scIndex + scLocalIndex) + if weightChangesDynamicReload && len(route.Splits) == 2 { + defaultResult = VariableNamer.GetNameOfMapForSplitClientIndex(scIndex + scLocalIndex) + } else { + defaultResult = VariableNamer.GetNameForSplitClientVariable(scIndex + scLocalIndex) + } } defaultParam := version2.Parameter{ @@ -2207,7 +2381,7 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr } params = append(params, defaultParam) - variable := variableNamer.GetNameForVariableForMatchesRouteMainMap(index) + variable := VariableNamer.GetNameForVariableForMatchesRouteMainMap(index) mainMap := version2.Map{ Source: source, @@ -2220,16 +2394,18 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr var locations []version2.Location var returnLocations []version2.ReturnLocation var splitClients []version2.SplitClient + var keyValZones []version2.KeyValZone + var keyVals []version2.KeyVal scLocalIndex = 0 for i, m := range route.Matches { if len(m.Splits) > 0 { newRetLocIndex := retLocIndex + len(returnLocations) - sc, locs, returnLocs := generateSplits( + scs, locs, returnLocs, mps, kvzs, kvs, twscs := generateSplits( m.Splits, upstreamNamer, crUpstreams, - variableNamer, + VariableNamer, scIndex+scLocalIndex, cfgParams, errorPages, @@ -2241,11 +2417,16 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr vsrName, vsrNamespace, vscWarnings, + weightChangesDynamicReload, ) - scLocalIndex++ - splitClients = append(splitClients, sc) + scLocalIndex += len(scs) + splitClients = append(splitClients, scs...) locations = append(locations, locs...) returnLocations = append(returnLocations, returnLocs...) + maps = append(maps, mps...) + keyValZones = append(keyValZones, kvzs...) + keyVals = append(keyVals, kvs...) + twoWaySplitClients = append(twoWaySplitClients, twscs...) } else { path := fmt.Sprintf("/%vmatches_%d_match_%d", internalLocationPrefix, index, i) upstreamName := upstreamNamer.GetNameForUpstreamFromAction(m.Action) @@ -2264,11 +2445,11 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr // Generate default splits or default action if len(route.Splits) > 0 { newRetLocIndex := retLocIndex + len(returnLocations) - sc, locs, returnLocs := generateSplits( + scs, locs, returnLocs, mps, kvzs, kvs, twscs := generateSplits( route.Splits, upstreamNamer, crUpstreams, - variableNamer, + VariableNamer, scIndex+scLocalIndex, cfgParams, errorPages, @@ -2280,10 +2461,15 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr vsrName, vsrNamespace, vscWarnings, + weightChangesDynamicReload, ) - splitClients = append(splitClients, sc) + splitClients = append(splitClients, scs...) locations = append(locations, locs...) returnLocations = append(returnLocations, returnLocs...) + maps = append(maps, mps...) + keyValZones = append(keyValZones, kvzs...) + keyVals = append(keyVals, kvs...) + twoWaySplitClients = append(twoWaySplitClients, twscs...) } else { path := fmt.Sprintf("/%vmatches_%d_default", internalLocationPrefix, index) upstreamName := upstreamNamer.GetNameForUpstreamFromAction(route.Action) @@ -2310,6 +2496,9 @@ func generateMatchesConfig(route conf_v1.Route, upstreamNamer *upstreamNamer, cr InternalRedirectLocation: irl, SplitClients: splitClients, ReturnLocations: returnLocations, + KeyValZones: keyValZones, + KeyVals: keyVals, + TwoWaySplitClients: twoWaySplitClients, } } diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index c072b07277..e05e9db4b5 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -157,7 +157,7 @@ func TestVariableNamerSafeNsName(t *testing.T) { expected := "default_cafe_test" - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) if variableNamer.safeNsName != expected { t.Errorf( @@ -176,7 +176,7 @@ func TestVariableNamer(t *testing.T) { Namespace: "default", }, } - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) // GetNameForSplitClientVariable() index := 0 @@ -8896,9 +8896,9 @@ func TestCreateUpstreamServersConfigForPlusNoUpstreams(t *testing.T) { func TestGenerateSplits(t *testing.T) { t.Parallel() tests := []struct { - splits []conf_v1.Split - expectedSplitClient version2.SplitClient - msg string + splits []conf_v1.Split + expectedSplitClients []version2.SplitClient + msg string }{ { splits: []conf_v1.Split{ @@ -8926,21 +8926,23 @@ func TestGenerateSplits(t *testing.T) { }, }, }, - expectedSplitClient: version2.SplitClient{ - Source: "$request_id", - Variable: "$vs_default_cafe_splits_1", - Distributions: []version2.Distribution{ - { - Weight: "90%", - Value: "/internal_location_splits_1_split_0", - }, - { - Weight: "9%", - Value: "/internal_location_splits_1_split_1", - }, - { - Weight: "1%", - Value: "/internal_location_splits_1_split_2", + expectedSplitClients: []version2.SplitClient{ + { + Source: "$request_id", + Variable: "$vs_default_cafe_splits_1", + Distributions: []version2.Distribution{ + { + Weight: "90%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "9%", + Value: "/internal_location_splits_1_split_1", + }, + { + Weight: "1%", + Value: "/internal_location_splits_1_split_2", + }, }, }, }, @@ -8972,17 +8974,19 @@ func TestGenerateSplits(t *testing.T) { }, }, }, - expectedSplitClient: version2.SplitClient{ - Source: "$request_id", - Variable: "$vs_default_cafe_splits_1", - Distributions: []version2.Distribution{ - { - Weight: "90%", - Value: "/internal_location_splits_1_split_0", - }, - { - Weight: "10%", - Value: "/internal_location_splits_1_split_2", + expectedSplitClients: []version2.SplitClient{ + { + Source: "$request_id", + Variable: "$vs_default_cafe_splits_1", + Distributions: []version2.Distribution{ + { + Weight: "90%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "10%", + Value: "/internal_location_splits_1_split_2", + }, }, }, }, @@ -8998,7 +9002,7 @@ func TestGenerateSplits(t *testing.T) { }, } upstreamNamer := NewUpstreamNamerForVirtualServer(&virtualServer) - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) scIndex := 1 cfgParams := ConfigParams{} crUpstreams := map[string]conf_v1.Upstream{ @@ -9137,7 +9141,7 @@ func TestGenerateSplits(t *testing.T) { vsc := newVirtualServerConfigurator(&cfgParams, false, false, &StaticConfigParams{}, false, &fakeBV) for _, test := range tests { t.Run(test.msg, func(t *testing.T) { - resultSplitClient, resultLocations, resultReturnLocations := generateSplits( + resultSplitClients, resultLocations, resultReturnLocations, _, _, _, _ := generateSplits( test.splits, upstreamNamer, crUpstreams, @@ -9153,10 +9157,11 @@ func TestGenerateSplits(t *testing.T) { "coffee", "default", vsc.warnings, + vsc.DynamicWeightChangesReload, ) - if !cmp.Equal(test.expectedSplitClient, resultSplitClient) { - t.Errorf("generateSplits() resultSplitClient mismatch (-want +got):\n%s", cmp.Diff(test.expectedSplitClient, resultSplitClient)) + if !cmp.Equal(test.expectedSplitClients, resultSplitClients) { + t.Errorf("generateSplits() resultSplitClient mismatch (-want +got):\n%s", cmp.Diff(test.expectedSplitClients, resultSplitClients)) } if !cmp.Equal(expectedLocations, resultLocations) { t.Errorf("generateSplits() resultLocations mismatch (-want +got):\n%s", cmp.Diff(expectedLocations, resultLocations)) @@ -9168,6 +9173,1690 @@ func TestGenerateSplits(t *testing.T) { } } +func TestGenerateSplitsWeightChangesDynamicReload(t *testing.T) { + t.Parallel() + tests := []struct { + splits []conf_v1.Split + expectedSplitClients []version2.SplitClient + msg string + }{ + { + splits: []conf_v1.Split{ + { + Weight: 90, + Action: &conf_v1.Action{ + Proxy: &conf_v1.ActionProxy{ + Upstream: "coffee-v1", + RewritePath: "/rewrite", + }, + }, + }, + { + Weight: 10, + Action: &conf_v1.Action{ + Pass: "coffee-v2", + }, + }, + }, + expectedSplitClients: []version2.SplitClient{ + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_0_100", + Distributions: []version2.Distribution{ + { + Weight: "100%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_1_99", + Distributions: []version2.Distribution{ + { + Weight: "1%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "99%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_2_98", + Distributions: []version2.Distribution{ + { + Weight: "2%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "98%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_3_97", + Distributions: []version2.Distribution{ + { + Weight: "3%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "97%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_4_96", + Distributions: []version2.Distribution{ + { + Weight: "4%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "96%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_5_95", + Distributions: []version2.Distribution{ + { + Weight: "5%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "95%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_6_94", + Distributions: []version2.Distribution{ + { + Weight: "6%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "94%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_7_93", + Distributions: []version2.Distribution{ + { + Weight: "7%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "93%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_8_92", + Distributions: []version2.Distribution{ + { + Weight: "8%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "92%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_9_91", + Distributions: []version2.Distribution{ + { + Weight: "9%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "91%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_10_90", + Distributions: []version2.Distribution{ + { + Weight: "10%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "90%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_11_89", + Distributions: []version2.Distribution{ + { + Weight: "11%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "89%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_12_88", + Distributions: []version2.Distribution{ + { + Weight: "12%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "88%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_13_87", + Distributions: []version2.Distribution{ + { + Weight: "13%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "87%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_14_86", + Distributions: []version2.Distribution{ + { + Weight: "14%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "86%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_15_85", + Distributions: []version2.Distribution{ + { + Weight: "15%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "85%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_16_84", + Distributions: []version2.Distribution{ + { + Weight: "16%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "84%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_17_83", + Distributions: []version2.Distribution{ + { + Weight: "17%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "83%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_18_82", + Distributions: []version2.Distribution{ + { + Weight: "18%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "82%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_19_81", + Distributions: []version2.Distribution{ + { + Weight: "19%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "81%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_20_80", + Distributions: []version2.Distribution{ + { + Weight: "20%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "80%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_21_79", + Distributions: []version2.Distribution{ + { + Weight: "21%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "79%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_22_78", + Distributions: []version2.Distribution{ + { + Weight: "22%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "78%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_23_77", + Distributions: []version2.Distribution{ + { + Weight: "23%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "77%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_24_76", + Distributions: []version2.Distribution{ + { + Weight: "24%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "76%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_25_75", + Distributions: []version2.Distribution{ + { + Weight: "25%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "75%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_26_74", + Distributions: []version2.Distribution{ + { + Weight: "26%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "74%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_27_73", + Distributions: []version2.Distribution{ + { + Weight: "27%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "73%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_28_72", + Distributions: []version2.Distribution{ + { + Weight: "28%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "72%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_29_71", + Distributions: []version2.Distribution{ + { + Weight: "29%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "71%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_30_70", + Distributions: []version2.Distribution{ + { + Weight: "30%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "70%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_31_69", + Distributions: []version2.Distribution{ + { + Weight: "31%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "69%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_32_68", + Distributions: []version2.Distribution{ + { + Weight: "32%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "68%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_33_67", + Distributions: []version2.Distribution{ + { + Weight: "33%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "67%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_34_66", + Distributions: []version2.Distribution{ + { + Weight: "34%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "66%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_35_65", + Distributions: []version2.Distribution{ + { + Weight: "35%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "65%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_36_64", + Distributions: []version2.Distribution{ + { + Weight: "36%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "64%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_37_63", + Distributions: []version2.Distribution{ + { + Weight: "37%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "63%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_38_62", + Distributions: []version2.Distribution{ + { + Weight: "38%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "62%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_39_61", + Distributions: []version2.Distribution{ + { + Weight: "39%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "61%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_40_60", + Distributions: []version2.Distribution{ + { + Weight: "40%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "60%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_41_59", + Distributions: []version2.Distribution{ + { + Weight: "41%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "59%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_42_58", + Distributions: []version2.Distribution{ + { + Weight: "42%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "58%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_43_57", + Distributions: []version2.Distribution{ + { + Weight: "43%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "57%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_44_56", + Distributions: []version2.Distribution{ + { + Weight: "44%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "56%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_45_55", + Distributions: []version2.Distribution{ + { + Weight: "45%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "55%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_46_54", + Distributions: []version2.Distribution{ + { + Weight: "46%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "54%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_47_53", + Distributions: []version2.Distribution{ + { + Weight: "47%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "53%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_48_52", + Distributions: []version2.Distribution{ + { + Weight: "48%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "52%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_49_51", + Distributions: []version2.Distribution{ + { + Weight: "49%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "51%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_50_50", + Distributions: []version2.Distribution{ + { + Weight: "50%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "50%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_51_49", + Distributions: []version2.Distribution{ + { + Weight: "51%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "49%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_52_48", + Distributions: []version2.Distribution{ + { + Weight: "52%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "48%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_53_47", + Distributions: []version2.Distribution{ + { + Weight: "53%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "47%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_54_46", + Distributions: []version2.Distribution{ + { + Weight: "54%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "46%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_55_45", + Distributions: []version2.Distribution{ + { + Weight: "55%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "45%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_56_44", + Distributions: []version2.Distribution{ + { + Weight: "56%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "44%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_57_43", + Distributions: []version2.Distribution{ + { + Weight: "57%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "43%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_58_42", + Distributions: []version2.Distribution{ + { + Weight: "58%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "42%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_59_41", + Distributions: []version2.Distribution{ + { + Weight: "59%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "41%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_60_40", + Distributions: []version2.Distribution{ + { + Weight: "60%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "40%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_61_39", + Distributions: []version2.Distribution{ + { + Weight: "61%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "39%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_62_38", + Distributions: []version2.Distribution{ + { + Weight: "62%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "38%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_63_37", + Distributions: []version2.Distribution{ + { + Weight: "63%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "37%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_64_36", + Distributions: []version2.Distribution{ + { + Weight: "64%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "36%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_65_35", + Distributions: []version2.Distribution{ + { + Weight: "65%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "35%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_66_34", + Distributions: []version2.Distribution{ + { + Weight: "66%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "34%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_67_33", + Distributions: []version2.Distribution{ + { + Weight: "67%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "33%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_68_32", + Distributions: []version2.Distribution{ + { + Weight: "68%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "32%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_69_31", + Distributions: []version2.Distribution{ + { + Weight: "69%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "31%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_70_30", + Distributions: []version2.Distribution{ + { + Weight: "70%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "30%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_71_29", + Distributions: []version2.Distribution{ + { + Weight: "71%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "29%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_72_28", + Distributions: []version2.Distribution{ + { + Weight: "72%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "28%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_73_27", + Distributions: []version2.Distribution{ + { + Weight: "73%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "27%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_74_26", + Distributions: []version2.Distribution{ + { + Weight: "74%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "26%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_75_25", + Distributions: []version2.Distribution{ + { + Weight: "75%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "25%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_76_24", + Distributions: []version2.Distribution{ + { + Weight: "76%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "24%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_77_23", + Distributions: []version2.Distribution{ + { + Weight: "77%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "23%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_78_22", + Distributions: []version2.Distribution{ + { + Weight: "78%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "22%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_79_21", + Distributions: []version2.Distribution{ + { + Weight: "79%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "21%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_80_20", + Distributions: []version2.Distribution{ + { + Weight: "80%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "20%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_81_19", + Distributions: []version2.Distribution{ + { + Weight: "81%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "19%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_82_18", + Distributions: []version2.Distribution{ + { + Weight: "82%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "18%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_83_17", + Distributions: []version2.Distribution{ + { + Weight: "83%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "17%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_84_16", + Distributions: []version2.Distribution{ + { + Weight: "84%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "16%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_85_15", + Distributions: []version2.Distribution{ + { + Weight: "85%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "15%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_86_14", + Distributions: []version2.Distribution{ + { + Weight: "86%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "14%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_87_13", + Distributions: []version2.Distribution{ + { + Weight: "87%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "13%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_88_12", + Distributions: []version2.Distribution{ + { + Weight: "88%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "12%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_89_11", + Distributions: []version2.Distribution{ + { + Weight: "89%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "11%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_90_10", + Distributions: []version2.Distribution{ + { + Weight: "90%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "10%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_91_9", + Distributions: []version2.Distribution{ + { + Weight: "91%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "9%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_92_8", + Distributions: []version2.Distribution{ + { + Weight: "92%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "8%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_93_7", + Distributions: []version2.Distribution{ + { + Weight: "93%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "7%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_94_6", + Distributions: []version2.Distribution{ + { + Weight: "94%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "6%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_95_5", + Distributions: []version2.Distribution{ + { + Weight: "95%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "5%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_96_4", + Distributions: []version2.Distribution{ + { + Weight: "96%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "4%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_97_3", + Distributions: []version2.Distribution{ + { + Weight: "97%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "3%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_98_2", + Distributions: []version2.Distribution{ + { + Weight: "98%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "2%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_99_1", + Distributions: []version2.Distribution{ + { + Weight: "99%", + Value: "/internal_location_splits_1_split_0", + }, + { + Weight: "1%", + Value: "/internal_location_splits_1_split_1", + }, + }, + }, + { + Source: "$request_id", + Variable: "$vs_default_cafe_split_clients_1_100_0", + Distributions: []version2.Distribution{ + { + Weight: "100%", + Value: "/internal_location_splits_1_split_0", + }, + }, + }, + }, + msg: "Normal Split", + }, + } + originalPath := "/path" + + virtualServer := conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + } + upstreamNamer := NewUpstreamNamerForVirtualServer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) + scIndex := 1 + cfgParams := ConfigParams{} + crUpstreams := map[string]conf_v1.Upstream{ + "vs_default_cafe_coffee-v1": { + Service: "coffee-v1", + }, + "vs_default_cafe_coffee-v2": { + Service: "coffee-v2", + }, + } + enableSnippets := false + expectedLocations := []version2.Location{ + { + Path: "/internal_location_splits_1_split_0", + ProxyPass: "http://vs_default_cafe_coffee-v1", + Rewrites: []string{ + "^ $request_uri_no_args", + fmt.Sprintf(`"^%v(.*)$" "/rewrite$1" break`, originalPath), + }, + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + Internal: true, + ProxySSLName: "coffee-v1.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "coffee-v1", + IsVSR: true, + VSRName: "coffee", + VSRNamespace: "default", + }, + { + Path: "/internal_location_splits_1_split_1", + ProxyPass: "http://vs_default_cafe_coffee-v2$request_uri", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "0s", + ProxyNextUpstreamTries: 0, + Internal: true, + ProxySSLName: "coffee-v2.default.svc", + ProxyPassRequestHeaders: true, + ProxySetHeaders: []version2.Header{{Name: "Host", Value: "$host"}}, + ServiceName: "coffee-v2", + IsVSR: true, + VSRName: "coffee", + VSRNamespace: "default", + }, + } + + expectedMaps := []version2.Map{ + { + Source: "$vs_default_cafe_keyval_split_clients_1", + Variable: "$vs_default_cafe_map_split_clients_1", + Parameters: []version2.Parameter{ + {Value: `"vs_default_cafe_split_clients_1_0_100"`, Result: "$vs_default_cafe_split_clients_1_0_100"}, + {Value: `"vs_default_cafe_split_clients_1_1_99"`, Result: "$vs_default_cafe_split_clients_1_1_99"}, + {Value: `"vs_default_cafe_split_clients_1_2_98"`, Result: "$vs_default_cafe_split_clients_1_2_98"}, + {Value: `"vs_default_cafe_split_clients_1_3_97"`, Result: "$vs_default_cafe_split_clients_1_3_97"}, + {Value: `"vs_default_cafe_split_clients_1_4_96"`, Result: "$vs_default_cafe_split_clients_1_4_96"}, + {Value: `"vs_default_cafe_split_clients_1_5_95"`, Result: "$vs_default_cafe_split_clients_1_5_95"}, + {Value: `"vs_default_cafe_split_clients_1_6_94"`, Result: "$vs_default_cafe_split_clients_1_6_94"}, + {Value: `"vs_default_cafe_split_clients_1_7_93"`, Result: "$vs_default_cafe_split_clients_1_7_93"}, + {Value: `"vs_default_cafe_split_clients_1_8_92"`, Result: "$vs_default_cafe_split_clients_1_8_92"}, + {Value: `"vs_default_cafe_split_clients_1_9_91"`, Result: "$vs_default_cafe_split_clients_1_9_91"}, + {Value: `"vs_default_cafe_split_clients_1_10_90"`, Result: "$vs_default_cafe_split_clients_1_10_90"}, + {Value: `"vs_default_cafe_split_clients_1_11_89"`, Result: "$vs_default_cafe_split_clients_1_11_89"}, + {Value: `"vs_default_cafe_split_clients_1_12_88"`, Result: "$vs_default_cafe_split_clients_1_12_88"}, + {Value: `"vs_default_cafe_split_clients_1_13_87"`, Result: "$vs_default_cafe_split_clients_1_13_87"}, + {Value: `"vs_default_cafe_split_clients_1_14_86"`, Result: "$vs_default_cafe_split_clients_1_14_86"}, + {Value: `"vs_default_cafe_split_clients_1_15_85"`, Result: "$vs_default_cafe_split_clients_1_15_85"}, + {Value: `"vs_default_cafe_split_clients_1_16_84"`, Result: "$vs_default_cafe_split_clients_1_16_84"}, + {Value: `"vs_default_cafe_split_clients_1_17_83"`, Result: "$vs_default_cafe_split_clients_1_17_83"}, + {Value: `"vs_default_cafe_split_clients_1_18_82"`, Result: "$vs_default_cafe_split_clients_1_18_82"}, + {Value: `"vs_default_cafe_split_clients_1_19_81"`, Result: "$vs_default_cafe_split_clients_1_19_81"}, + {Value: `"vs_default_cafe_split_clients_1_20_80"`, Result: "$vs_default_cafe_split_clients_1_20_80"}, + {Value: `"vs_default_cafe_split_clients_1_21_79"`, Result: "$vs_default_cafe_split_clients_1_21_79"}, + {Value: `"vs_default_cafe_split_clients_1_22_78"`, Result: "$vs_default_cafe_split_clients_1_22_78"}, + {Value: `"vs_default_cafe_split_clients_1_23_77"`, Result: "$vs_default_cafe_split_clients_1_23_77"}, + {Value: `"vs_default_cafe_split_clients_1_24_76"`, Result: "$vs_default_cafe_split_clients_1_24_76"}, + {Value: `"vs_default_cafe_split_clients_1_25_75"`, Result: "$vs_default_cafe_split_clients_1_25_75"}, + {Value: `"vs_default_cafe_split_clients_1_26_74"`, Result: "$vs_default_cafe_split_clients_1_26_74"}, + {Value: `"vs_default_cafe_split_clients_1_27_73"`, Result: "$vs_default_cafe_split_clients_1_27_73"}, + {Value: `"vs_default_cafe_split_clients_1_28_72"`, Result: "$vs_default_cafe_split_clients_1_28_72"}, + {Value: `"vs_default_cafe_split_clients_1_29_71"`, Result: "$vs_default_cafe_split_clients_1_29_71"}, + {Value: `"vs_default_cafe_split_clients_1_30_70"`, Result: "$vs_default_cafe_split_clients_1_30_70"}, + {Value: `"vs_default_cafe_split_clients_1_31_69"`, Result: "$vs_default_cafe_split_clients_1_31_69"}, + {Value: `"vs_default_cafe_split_clients_1_32_68"`, Result: "$vs_default_cafe_split_clients_1_32_68"}, + {Value: `"vs_default_cafe_split_clients_1_33_67"`, Result: "$vs_default_cafe_split_clients_1_33_67"}, + {Value: `"vs_default_cafe_split_clients_1_34_66"`, Result: "$vs_default_cafe_split_clients_1_34_66"}, + {Value: `"vs_default_cafe_split_clients_1_35_65"`, Result: "$vs_default_cafe_split_clients_1_35_65"}, + {Value: `"vs_default_cafe_split_clients_1_36_64"`, Result: "$vs_default_cafe_split_clients_1_36_64"}, + {Value: `"vs_default_cafe_split_clients_1_37_63"`, Result: "$vs_default_cafe_split_clients_1_37_63"}, + {Value: `"vs_default_cafe_split_clients_1_38_62"`, Result: "$vs_default_cafe_split_clients_1_38_62"}, + {Value: `"vs_default_cafe_split_clients_1_39_61"`, Result: "$vs_default_cafe_split_clients_1_39_61"}, + {Value: `"vs_default_cafe_split_clients_1_40_60"`, Result: "$vs_default_cafe_split_clients_1_40_60"}, + {Value: `"vs_default_cafe_split_clients_1_41_59"`, Result: "$vs_default_cafe_split_clients_1_41_59"}, + {Value: `"vs_default_cafe_split_clients_1_42_58"`, Result: "$vs_default_cafe_split_clients_1_42_58"}, + {Value: `"vs_default_cafe_split_clients_1_43_57"`, Result: "$vs_default_cafe_split_clients_1_43_57"}, + {Value: `"vs_default_cafe_split_clients_1_44_56"`, Result: "$vs_default_cafe_split_clients_1_44_56"}, + {Value: `"vs_default_cafe_split_clients_1_45_55"`, Result: "$vs_default_cafe_split_clients_1_45_55"}, + {Value: `"vs_default_cafe_split_clients_1_46_54"`, Result: "$vs_default_cafe_split_clients_1_46_54"}, + {Value: `"vs_default_cafe_split_clients_1_47_53"`, Result: "$vs_default_cafe_split_clients_1_47_53"}, + {Value: `"vs_default_cafe_split_clients_1_48_52"`, Result: "$vs_default_cafe_split_clients_1_48_52"}, + {Value: `"vs_default_cafe_split_clients_1_49_51"`, Result: "$vs_default_cafe_split_clients_1_49_51"}, + {Value: `"vs_default_cafe_split_clients_1_50_50"`, Result: "$vs_default_cafe_split_clients_1_50_50"}, + {Value: `"vs_default_cafe_split_clients_1_51_49"`, Result: "$vs_default_cafe_split_clients_1_51_49"}, + {Value: `"vs_default_cafe_split_clients_1_52_48"`, Result: "$vs_default_cafe_split_clients_1_52_48"}, + {Value: `"vs_default_cafe_split_clients_1_53_47"`, Result: "$vs_default_cafe_split_clients_1_53_47"}, + {Value: `"vs_default_cafe_split_clients_1_54_46"`, Result: "$vs_default_cafe_split_clients_1_54_46"}, + {Value: `"vs_default_cafe_split_clients_1_55_45"`, Result: "$vs_default_cafe_split_clients_1_55_45"}, + {Value: `"vs_default_cafe_split_clients_1_56_44"`, Result: "$vs_default_cafe_split_clients_1_56_44"}, + {Value: `"vs_default_cafe_split_clients_1_57_43"`, Result: "$vs_default_cafe_split_clients_1_57_43"}, + {Value: `"vs_default_cafe_split_clients_1_58_42"`, Result: "$vs_default_cafe_split_clients_1_58_42"}, + {Value: `"vs_default_cafe_split_clients_1_59_41"`, Result: "$vs_default_cafe_split_clients_1_59_41"}, + {Value: `"vs_default_cafe_split_clients_1_60_40"`, Result: "$vs_default_cafe_split_clients_1_60_40"}, + {Value: `"vs_default_cafe_split_clients_1_61_39"`, Result: "$vs_default_cafe_split_clients_1_61_39"}, + {Value: `"vs_default_cafe_split_clients_1_62_38"`, Result: "$vs_default_cafe_split_clients_1_62_38"}, + {Value: `"vs_default_cafe_split_clients_1_63_37"`, Result: "$vs_default_cafe_split_clients_1_63_37"}, + {Value: `"vs_default_cafe_split_clients_1_64_36"`, Result: "$vs_default_cafe_split_clients_1_64_36"}, + {Value: `"vs_default_cafe_split_clients_1_65_35"`, Result: "$vs_default_cafe_split_clients_1_65_35"}, + {Value: `"vs_default_cafe_split_clients_1_66_34"`, Result: "$vs_default_cafe_split_clients_1_66_34"}, + {Value: `"vs_default_cafe_split_clients_1_67_33"`, Result: "$vs_default_cafe_split_clients_1_67_33"}, + {Value: `"vs_default_cafe_split_clients_1_68_32"`, Result: "$vs_default_cafe_split_clients_1_68_32"}, + {Value: `"vs_default_cafe_split_clients_1_69_31"`, Result: "$vs_default_cafe_split_clients_1_69_31"}, + {Value: `"vs_default_cafe_split_clients_1_70_30"`, Result: "$vs_default_cafe_split_clients_1_70_30"}, + {Value: `"vs_default_cafe_split_clients_1_71_29"`, Result: "$vs_default_cafe_split_clients_1_71_29"}, + {Value: `"vs_default_cafe_split_clients_1_72_28"`, Result: "$vs_default_cafe_split_clients_1_72_28"}, + {Value: `"vs_default_cafe_split_clients_1_73_27"`, Result: "$vs_default_cafe_split_clients_1_73_27"}, + {Value: `"vs_default_cafe_split_clients_1_74_26"`, Result: "$vs_default_cafe_split_clients_1_74_26"}, + {Value: `"vs_default_cafe_split_clients_1_75_25"`, Result: "$vs_default_cafe_split_clients_1_75_25"}, + {Value: `"vs_default_cafe_split_clients_1_76_24"`, Result: "$vs_default_cafe_split_clients_1_76_24"}, + {Value: `"vs_default_cafe_split_clients_1_77_23"`, Result: "$vs_default_cafe_split_clients_1_77_23"}, + {Value: `"vs_default_cafe_split_clients_1_78_22"`, Result: "$vs_default_cafe_split_clients_1_78_22"}, + {Value: `"vs_default_cafe_split_clients_1_79_21"`, Result: "$vs_default_cafe_split_clients_1_79_21"}, + {Value: `"vs_default_cafe_split_clients_1_80_20"`, Result: "$vs_default_cafe_split_clients_1_80_20"}, + {Value: `"vs_default_cafe_split_clients_1_81_19"`, Result: "$vs_default_cafe_split_clients_1_81_19"}, + {Value: `"vs_default_cafe_split_clients_1_82_18"`, Result: "$vs_default_cafe_split_clients_1_82_18"}, + {Value: `"vs_default_cafe_split_clients_1_83_17"`, Result: "$vs_default_cafe_split_clients_1_83_17"}, + {Value: `"vs_default_cafe_split_clients_1_84_16"`, Result: "$vs_default_cafe_split_clients_1_84_16"}, + {Value: `"vs_default_cafe_split_clients_1_85_15"`, Result: "$vs_default_cafe_split_clients_1_85_15"}, + {Value: `"vs_default_cafe_split_clients_1_86_14"`, Result: "$vs_default_cafe_split_clients_1_86_14"}, + {Value: `"vs_default_cafe_split_clients_1_87_13"`, Result: "$vs_default_cafe_split_clients_1_87_13"}, + {Value: `"vs_default_cafe_split_clients_1_88_12"`, Result: "$vs_default_cafe_split_clients_1_88_12"}, + {Value: `"vs_default_cafe_split_clients_1_89_11"`, Result: "$vs_default_cafe_split_clients_1_89_11"}, + {Value: `"vs_default_cafe_split_clients_1_90_10"`, Result: "$vs_default_cafe_split_clients_1_90_10"}, + {Value: `"vs_default_cafe_split_clients_1_91_9"`, Result: "$vs_default_cafe_split_clients_1_91_9"}, + {Value: `"vs_default_cafe_split_clients_1_92_8"`, Result: "$vs_default_cafe_split_clients_1_92_8"}, + {Value: `"vs_default_cafe_split_clients_1_93_7"`, Result: "$vs_default_cafe_split_clients_1_93_7"}, + {Value: `"vs_default_cafe_split_clients_1_94_6"`, Result: "$vs_default_cafe_split_clients_1_94_6"}, + {Value: `"vs_default_cafe_split_clients_1_95_5"`, Result: "$vs_default_cafe_split_clients_1_95_5"}, + {Value: `"vs_default_cafe_split_clients_1_96_4"`, Result: "$vs_default_cafe_split_clients_1_96_4"}, + {Value: `"vs_default_cafe_split_clients_1_97_3"`, Result: "$vs_default_cafe_split_clients_1_97_3"}, + {Value: `"vs_default_cafe_split_clients_1_98_2"`, Result: "$vs_default_cafe_split_clients_1_98_2"}, + {Value: `"vs_default_cafe_split_clients_1_99_1"`, Result: "$vs_default_cafe_split_clients_1_99_1"}, + {Value: `"vs_default_cafe_split_clients_1_100_0"`, Result: "$vs_default_cafe_split_clients_1_100_0"}, + {Value: "default", Result: "$vs_default_cafe_split_clients_1_100_0"}, + }, + }, + } + + expectedKeyValZones := []version2.KeyValZone{ + { + Name: "vs_default_cafe_keyval_zone_split_clients_1", + Size: "100k", + State: "/etc/nginx/state_files/vs_default_cafe_keyval_zone_split_clients_1.json", + }, + } + + expectedKeyVals := []version2.KeyVal{ + { + Key: `"vs_default_cafe_keyval_key_split_clients_1"`, + Variable: "$vs_default_cafe_keyval_split_clients_1", + ZoneName: "vs_default_cafe_keyval_zone_split_clients_1", + }, + } + + expectedTwoWaySplitClients := []version2.TwoWaySplitClients{ + { + Key: `"vs_default_cafe_keyval_key_split_clients_1"`, + Variable: "$vs_default_cafe_keyval_split_clients_1", + ZoneName: "vs_default_cafe_keyval_zone_split_clients_1", + SplitClientsIndex: 1, + Weights: []int{90, 10}, + }, + } + returnLocationIndex := 1 + + staticConfigParams := &StaticConfigParams{ + DynamicWeightChangesReload: true, + } + + vsc := newVirtualServerConfigurator(&cfgParams, true, false, staticConfigParams, false, &fakeBV) + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + resultSplitClients, resultLocations, _, resultMaps, resultKeyValZones, resultKeyVals, resultTwoWaySplitClients := generateSplits( + test.splits, + upstreamNamer, + crUpstreams, + variableNamer, + scIndex, + &cfgParams, + errorPageDetails{}, + originalPath, + "", + enableSnippets, + returnLocationIndex, + true, + "coffee", + "default", + vsc.warnings, + vsc.DynamicWeightChangesReload, + ) + + if !cmp.Equal(test.expectedSplitClients, resultSplitClients) { + t.Errorf("generateSplits() resultSplitClient mismatch (-want +got):\n%s", cmp.Diff(test.expectedSplitClients, resultSplitClients)) + } + if !cmp.Equal(expectedLocations, resultLocations) { + t.Errorf("generateSplits() resultLocations mismatch (-want +got):\n%s", cmp.Diff(expectedLocations, resultLocations)) + } + + if !cmp.Equal(expectedMaps, resultMaps) { + t.Errorf("generateSplits() resultLocations mismatch (-want +got):\n%s", cmp.Diff(expectedMaps, resultMaps)) + } + + if !cmp.Equal(expectedKeyValZones, resultKeyValZones) { + t.Errorf("generateSplits() resultKeyValZones mismatch (-want +got):\n%s", cmp.Diff(expectedKeyValZones, resultKeyValZones)) + } + + if !cmp.Equal(expectedKeyVals, resultKeyVals) { + t.Errorf("generateSplits() resultKeyVals mismatch (-want +got):\n%s", cmp.Diff(expectedKeyVals, resultKeyVals)) + } + + if !cmp.Equal(expectedTwoWaySplitClients, resultTwoWaySplitClients) { + t.Errorf("generateSplits() resultTwoWaySplitClients mismatch (-want +got):\n%s", cmp.Diff(expectedTwoWaySplitClients, resultTwoWaySplitClients)) + } + }) + } +} + func TestGenerateDefaultSplitsConfig(t *testing.T) { t.Parallel() route := conf_v1.Route{ @@ -9194,7 +10883,7 @@ func TestGenerateDefaultSplitsConfig(t *testing.T) { }, } upstreamNamer := NewUpstreamNamerForVirtualServer(&virtualServer) - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) index := 1 expected := routingCfg{ @@ -9255,6 +10944,7 @@ func TestGenerateDefaultSplitsConfig(t *testing.T) { cfgParams := ConfigParams{} locSnippet := "" enableSnippets := false + weightChangesDynamicReload := false crUpstreams := map[string]conf_v1.Upstream{ "vs_default_cafe_coffee-v1": { Service: "coffee-v1", @@ -9271,7 +10961,7 @@ func TestGenerateDefaultSplitsConfig(t *testing.T) { } result := generateDefaultSplitsConfig(route, upstreamNamer, crUpstreams, variableNamer, index, &cfgParams, - errorPageDetails, "", locSnippet, enableSnippets, 0, true, "coffee", "default", Warnings{}) + errorPageDetails, "", locSnippet, enableSnippets, 0, true, "coffee", "default", Warnings{}, weightChangesDynamicReload) if !reflect.DeepEqual(result, expected) { t.Errorf("generateDefaultSplitsConfig() returned \n%+v but expected \n%+v", result, expected) } @@ -9380,7 +11070,7 @@ func TestGenerateMatchesConfig(t *testing.T) { }, } upstreamNamer := NewUpstreamNamerForVirtualServer(&virtualServer) - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) index := 1 scIndex := 2 @@ -9655,6 +11345,7 @@ func TestGenerateMatchesConfig(t *testing.T) { cfgParams := ConfigParams{} enableSnippets := false + weightChangesDynamicReload := false locSnippets := "" crUpstreams := map[string]conf_v1.Upstream{ "vs_default_cafe_coffee-v1": {Service: "coffee-v1"}, @@ -9684,6 +11375,7 @@ func TestGenerateMatchesConfig(t *testing.T) { "", "", Warnings{}, + weightChangesDynamicReload, ) if !reflect.DeepEqual(result, expected) { t.Errorf("generateMatchesConfig() returned \n%+v but expected \n%+v", result, expected) @@ -9762,7 +11454,7 @@ func TestGenerateMatchesConfigWithMultipleSplits(t *testing.T) { }, } upstreamNamer := NewUpstreamNamerForVirtualServer(&virtualServer) - variableNamer := newVariableNamer(&virtualServer) + variableNamer := NewVSVariableNamer(&virtualServer) index := 1 scIndex := 2 errorPages := []conf_v1.ErrorPage{ @@ -10066,6 +11758,7 @@ func TestGenerateMatchesConfigWithMultipleSplits(t *testing.T) { cfgParams := ConfigParams{} enableSnippets := false + weightChangesWithoutReload := false locSnippets := "" crUpstreams := map[string]conf_v1.Upstream{ "vs_default_cafe_coffee-v1": {Service: "coffee-v1"}, @@ -10094,6 +11787,7 @@ func TestGenerateMatchesConfigWithMultipleSplits(t *testing.T) { "coffee", "default", Warnings{}, + weightChangesWithoutReload, ) if !reflect.DeepEqual(result, expected) { t.Errorf("generateMatchesConfig() returned \n%+v but expected \n%+v", result, expected) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 8009fcb7fa..6470fe518f 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -78,8 +78,9 @@ const ( // IngressControllerName holds Ingress Controller name IngressControllerName = "nginx.org/ingress-controller" - typeKeyword = "type" - helmReleaseType = "helm.sh/release.v1" + typeKeyword = "type" + helmReleaseType = "helm.sh/release.v1" + splitClientAmountWhenWeightChangesDynamicReload = 101 ) var ( @@ -166,6 +167,7 @@ type LoadBalancerController struct { namespaceWatcherController cache.Controller telemetryCollector *telemetry.Collector telemetryChan chan struct{} + weightChangesDynamicReload bool } var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc @@ -213,6 +215,7 @@ type NewLoadBalancerControllerInput struct { WatchNamespaceLabel string EnableTelemetryReporting bool NICVersion string + DynamicWeightChangesReload bool } // NewLoadBalancerController creates a controller @@ -245,6 +248,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc isPrometheusEnabled: input.IsPrometheusEnabled, isLatencyMetricsEnabled: input.IsLatencyMetricsEnabled, isIPV6Disabled: input.IsIPV6Disabled, + weightChangesDynamicReload: input.DynamicWeightChangesReload, } eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) @@ -4279,3 +4283,216 @@ func (lbc *LoadBalancerController) addInternalRouteServer() { } } } + +func (lbc *LoadBalancerController) processVSWeightChangesDynamicReload(vsOld *conf_v1.VirtualServer, vsNew *conf_v1.VirtualServer) { + if lbc.haltIfVSConfigInvalid(vsNew) { + return + } + + var weightUpdates []configs.WeightUpdate + var splitClientsIndex int + variableNamer := configs.NewVSVariableNamer(vsNew) + + for i, routeNew := range vsNew.Spec.Routes { + routeOld := vsOld.Spec.Routes[i] + for j, matchNew := range routeNew.Matches { + matchOld := routeOld.Matches[j] + if len(matchNew.Splits) == 2 { + if matchNew.Splits[0].Weight != matchOld.Splits[0].Weight && matchNew.Splits[1].Weight != matchOld.Splits[1].Weight { + weightUpdates = append(weightUpdates, configs.WeightUpdate{ + Zone: variableNamer.GetNameOfKeyvalZoneForSplitClientIndex(splitClientsIndex), + Key: variableNamer.GetNameOfKeyvalKeyForSplitClientIndex(splitClientsIndex), + Value: variableNamer.GetNameOfKeyOfMapForWeights(splitClientsIndex, matchNew.Splits[0].Weight, matchNew.Splits[1].Weight), + }) + } + splitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(matchNew.Splits) > 0 { + splitClientsIndex++ + } + } + if len(routeNew.Splits) == 2 { + if routeNew.Splits[0].Weight != routeOld.Splits[0].Weight && routeNew.Splits[1].Weight != routeOld.Splits[1].Weight { + weightUpdates = append(weightUpdates, configs.WeightUpdate{ + Zone: variableNamer.GetNameOfKeyvalZoneForSplitClientIndex(splitClientsIndex), + Key: variableNamer.GetNameOfKeyvalKeyForSplitClientIndex(splitClientsIndex), + Value: variableNamer.GetNameOfKeyOfMapForWeights(splitClientsIndex, routeNew.Splits[0].Weight, routeNew.Splits[1].Weight), + }) + splitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } + splitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(routeNew.Splits) > 0 { + splitClientsIndex++ + } + } + for _, weight := range weightUpdates { + lbc.configurator.UpsertSplitClientsKeyVal(weight.Zone, weight.Key, weight.Value) + } +} + +func (lbc *LoadBalancerController) processVSRWeightChangesDynamicReload(vsrOld *conf_v1.VirtualServerRoute, vsrNew *conf_v1.VirtualServerRoute) { + halt, vsEx := lbc.haltIfVSRConfigInvalid(vsrNew) + if halt { + return + } + + var weightUpdates []configs.WeightUpdate + + splitClientsIndex := lbc.getStartingSplitClientsIndex(vsrNew, vsEx) + + variableNamer := configs.NewVSVariableNamer(vsEx.VirtualServer) + + for i, routeNew := range vsrNew.Spec.Subroutes { + routeOld := vsrOld.Spec.Subroutes[i] + for j, matchNew := range routeNew.Matches { + matchOld := routeOld.Matches[j] + if len(matchNew.Splits) == 2 { + if matchNew.Splits[0].Weight != matchOld.Splits[0].Weight && matchNew.Splits[1].Weight != matchOld.Splits[1].Weight { + weightUpdates = append(weightUpdates, configs.WeightUpdate{ + Zone: variableNamer.GetNameOfKeyvalZoneForSplitClientIndex(splitClientsIndex), + Key: variableNamer.GetNameOfKeyvalKeyForSplitClientIndex(splitClientsIndex), + Value: variableNamer.GetNameOfKeyOfMapForWeights(splitClientsIndex, matchNew.Splits[0].Weight, matchNew.Splits[1].Weight), + }) + } + splitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(matchNew.Splits) > 0 { + splitClientsIndex++ + } + } + if len(routeNew.Splits) == 2 { + if routeNew.Splits[0].Weight != routeOld.Splits[0].Weight && routeNew.Splits[1].Weight != routeOld.Splits[1].Weight { + weightUpdates = append(weightUpdates, configs.WeightUpdate{ + Zone: variableNamer.GetNameOfKeyvalZoneForSplitClientIndex(splitClientsIndex), + Key: variableNamer.GetNameOfKeyvalKeyForSplitClientIndex(splitClientsIndex), + Value: variableNamer.GetNameOfKeyOfMapForWeights(splitClientsIndex, routeNew.Splits[0].Weight, routeNew.Splits[1].Weight), + }) + } + splitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(routeNew.Splits) > 0 { + splitClientsIndex++ + } + } + for _, weight := range weightUpdates { + lbc.configurator.UpsertSplitClientsKeyVal(weight.Zone, weight.Key, weight.Value) + } +} + +func (lbc *LoadBalancerController) getStartingSplitClientsIndex(vsr *conf_v1.VirtualServerRoute, vsEx *configs.VirtualServerEx) int { + var startingSplitClientsIndex int + + for _, r := range vsEx.VirtualServer.Spec.Routes { + for _, match := range r.Matches { + if len(match.Splits) == 2 { + startingSplitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(match.Splits) > 0 { + startingSplitClientsIndex++ + } + } + if len(r.Splits) == 2 { + startingSplitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(r.Splits) > 0 { + startingSplitClientsIndex++ + } + + } + + for _, vsRoute := range vsEx.VirtualServerRoutes { + if vsRoute.Name == vsr.Name { + return startingSplitClientsIndex + } + for _, r := range vsRoute.Spec.Subroutes { + for _, match := range r.Matches { + if len(match.Splits) == 2 { + startingSplitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(match.Splits) > 0 { + startingSplitClientsIndex++ + } + } + if len(r.Splits) == 2 { + startingSplitClientsIndex += splitClientAmountWhenWeightChangesDynamicReload + } else if len(r.Splits) > 0 { + startingSplitClientsIndex++ + } + } + } + + return startingSplitClientsIndex +} + +func (lbc *LoadBalancerController) haltIfVSConfigInvalid(vsNew *conf_v1.VirtualServer) bool { + lbc.configuration.lock.Lock() + defer lbc.configuration.lock.Unlock() + key := getResourceKey(&vsNew.ObjectMeta) + + validationError := lbc.configuration.virtualServerValidator.ValidateVirtualServer(vsNew) + if validationError != nil { + delete(lbc.configuration.virtualServers, key) + } else { + lbc.configuration.virtualServers[key] = vsNew + } + + changes, problems := lbc.configuration.rebuildHosts() + + if len(problems) > 0 { + lbc.processProblems(problems) + return true + } + + if len(changes) == 0 { + return true + } + + for _, c := range changes { + if c.Op == AddOrUpdate { + switch impl := c.Resource.(type) { + case *VirtualServerConfiguration: + lbc.updateVirtualServerStatusAndEvents(impl, configs.Warnings{}, nil) + } + } + } + + lbc.configuration.virtualServers[key] = vsNew + return false +} + +func (lbc *LoadBalancerController) haltIfVSRConfigInvalid(vsrNew *conf_v1.VirtualServerRoute) (bool, *configs.VirtualServerEx) { + lbc.configuration.lock.Lock() + defer lbc.configuration.lock.Unlock() + key := getResourceKey(&vsrNew.ObjectMeta) + var vsEx *configs.VirtualServerEx + + validationError := lbc.configuration.virtualServerValidator.ValidateVirtualServerRoute(vsrNew) + if validationError != nil { + delete(lbc.configuration.virtualServerRoutes, key) + } else { + lbc.configuration.virtualServerRoutes[key] = vsrNew + } + + changes, problems := lbc.configuration.rebuildHosts() + + if len(problems) > 0 { + lbc.processProblems(problems) + return true, nil + } + + if len(changes) == 0 { + return true, nil + } + + for _, c := range changes { + if c.Op == AddOrUpdate { + switch impl := c.Resource.(type) { + case *VirtualServerConfiguration: + vsEx = lbc.createVirtualServerEx(impl.VirtualServer, impl.VirtualServerRoutes) + lbc.updateVirtualServerStatusAndEvents(impl, configs.Warnings{}, nil) + } + } + } + + if vsEx == nil { + glog.V(3).Infof("VirtualServerRoute %s does not have a corresponding VirtualServer", vsrNew.Name) + return true, nil + } + + lbc.configuration.virtualServerRoutes[key] = vsrNew + return false, vsEx +} diff --git a/internal/k8s/handlers.go b/internal/k8s/handlers.go index 88f06a7766..bb4c35bf03 100644 --- a/internal/k8s/handlers.go +++ b/internal/k8s/handlers.go @@ -7,6 +7,7 @@ import ( discovery_v1 "k8s.io/api/discovery/v1" + "github.com/jinzhu/copier" "github.com/nginxinc/kubernetes-ingress/pkg/apis/dos/v1beta1" "github.com/golang/glog" @@ -309,6 +310,31 @@ func createVirtualServerHandlers(lbc *LoadBalancerController) cache.ResourceEven UpdateFunc: func(old, cur interface{}) { curVs := cur.(*conf_v1.VirtualServer) oldVs := old.(*conf_v1.VirtualServer) + + if lbc.weightChangesDynamicReload { + var curVsCopy, oldVsCopy conf_v1.VirtualServer + err := copier.CopyWithOption(&curVsCopy, curVs, copier.Option{DeepCopy: true}) + if err != nil { + glog.V(3).Infof("Error copying VirtualServer %v: %v for Dynamic Weight Changes", curVs.Name, err) + return + } + + err = copier.CopyWithOption(&oldVsCopy, oldVs, copier.Option{DeepCopy: true}) + if err != nil { + glog.V(3).Infof("Error copying VirtualServer %v: %v for Dynamic Weight Changes", oldVs.Name, err) + return + } + + zeroOutVirtualServerSplitWeights(&curVsCopy) + zeroOutVirtualServerSplitWeights(&oldVsCopy) + + if reflect.DeepEqual(oldVsCopy.Spec, curVsCopy.Spec) { + lbc.processVSWeightChangesDynamicReload(oldVs, curVs) + return + } + + } + if !reflect.DeepEqual(oldVs.Spec, curVs.Spec) { glog.V(3).Infof("VirtualServer %v changed, syncing", curVs.Name) lbc.AddSyncQueue(curVs) @@ -344,6 +370,31 @@ func createVirtualServerRouteHandlers(lbc *LoadBalancerController) cache.Resourc UpdateFunc: func(old, cur interface{}) { curVsr := cur.(*conf_v1.VirtualServerRoute) oldVsr := old.(*conf_v1.VirtualServerRoute) + + if lbc.weightChangesDynamicReload { + var curVsrCopy, oldVsrCopy conf_v1.VirtualServerRoute + err := copier.CopyWithOption(&curVsrCopy, curVsr, copier.Option{DeepCopy: true}) + if err != nil { + glog.V(3).Infof("Error copying VirtualServerRoute %v: %v for Dynamic Weight Changes", curVsr.Name, err) + return + } + + err = copier.CopyWithOption(&oldVsrCopy, oldVsr, copier.Option{DeepCopy: true}) + if err != nil { + glog.V(3).Infof("Error copying VirtualServerRoute %v: %v for Dynamic Weight Changes", oldVsr.Name, err) + return + } + + zeroOutVirtualServerRouteSplitWeights(&curVsrCopy) + zeroOutVirtualServerRouteSplitWeights(&oldVsrCopy) + + if reflect.DeepEqual(oldVsrCopy.Spec, curVsrCopy.Spec) { + lbc.processVSRWeightChangesDynamicReload(oldVsr, curVsr) + return + } + + } + if !reflect.DeepEqual(oldVsr.Spec, curVsr.Spec) { glog.V(3).Infof("VirtualServerRoute %v changed, syncing", curVsr.Name) lbc.AddSyncQueue(curVsr) @@ -711,3 +762,35 @@ func createNamespaceHandlers(lbc *LoadBalancerController) cache.ResourceEventHan }, } } + +func zeroOutVirtualServerSplitWeights(vs *conf_v1.VirtualServer) { + for _, route := range vs.Spec.Routes { + for _, match := range route.Matches { + if len(match.Splits) == 2 { + match.Splits[0].Weight = 0 + match.Splits[1].Weight = 0 + } + } + + if len(route.Splits) == 2 { + route.Splits[0].Weight = 0 + route.Splits[1].Weight = 0 + } + } +} + +func zeroOutVirtualServerRouteSplitWeights(vs *conf_v1.VirtualServerRoute) { + for _, route := range vs.Spec.Subroutes { + for _, match := range route.Matches { + if len(match.Splits) == 2 { + match.Splits[0].Weight = 0 + match.Splits[1].Weight = 0 + } + } + + if len(route.Splits) == 2 { + route.Splits[0].Weight = 0 + route.Splits[1].Weight = 0 + } + } +} diff --git a/internal/nginx/fake_manager.go b/internal/nginx/fake_manager.go index b52e3d2f65..2aafffc513 100644 --- a/internal/nginx/fake_manager.go +++ b/internal/nginx/fake_manager.go @@ -193,3 +193,13 @@ func (*FakeManager) AgentVersion() string { func (fm *FakeManager) GetSecretsDir() string { return fm.secretsPath } + +// UpsertSplitClientsKeyVal is a fake implementation of UpsertSplitClientsKeyVal +func (fm *FakeManager) UpsertSplitClientsKeyVal(_ string, _ string, _ string) { + glog.V(3).Infof("Creating split clients key") +} + +// DeleteKeyValStateFiles is a fake implementation of DeleteKeyValStateFiles +func (fm *FakeManager) DeleteKeyValStateFiles(_ string) { + glog.V(3).Infof("Deleting keyval state files") +} diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index f23918856c..18fe1ea88f 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -96,6 +96,8 @@ type Manager interface { AgentQuit() AgentVersion() string GetSecretsDir() string + UpsertSplitClientsKeyVal(zoneName string, key string, value string) + DeleteKeyValStateFiles(virtualServerName string) } // LocalManager updates NGINX configuration, starts, reloads and quits NGINX, @@ -104,6 +106,7 @@ type LocalManager struct { confdPath string streamConfdPath string secretsPath string + stateFilesPath string mainConfFilename string configVersionFilename string debug bool @@ -132,6 +135,7 @@ func NewLocalManager(confPath string, debug bool, mc collectors.ManagerCollector confdPath: path.Join(confPath, "conf.d"), streamConfdPath: path.Join(confPath, "stream-conf.d"), secretsPath: path.Join(confPath, "secrets"), + stateFilesPath: path.Join(confPath, "state_files"), dhparamFilename: path.Join(confPath, "secrets", "dhparam.pem"), mainConfFilename: path.Join(confPath, "nginx.conf"), configVersionFilename: path.Join(confPath, "config-version.conf"), @@ -628,3 +632,54 @@ func configContentsChanged(filename string, content []byte) bool { } return true } + +// UpsertSplitClientsKeyVal upserts a key value pair in the split clients zone. +func (lm *LocalManager) UpsertSplitClientsKeyVal(zoneName, key, value string) { + key = strings.Trim(key, "\"") + value = strings.Trim(value, "\"") + + keyValPairs, err := lm.plusClient.GetKeyValPairs(zoneName) + if err != nil { + lm.tryAddKeyValPair(zoneName, key, value) + return + } + + if _, ok := keyValPairs[key]; ok { + lm.tryModifyKeyValPair(zoneName, key, value) + } else { + lm.tryAddKeyValPair(zoneName, key, value) + } +} + +func (lm *LocalManager) tryAddKeyValPair(zoneName, key, value string) { + err := lm.plusClient.AddKeyValPair(zoneName, key, value) + if err != nil { + glog.Warningf("Failed to add key value pair: %v", err) + } else { + glog.Infof("Added key value pair for key: %v", key) + } +} + +func (lm *LocalManager) tryModifyKeyValPair(zoneName, key, value string) { + err := lm.plusClient.ModifyKeyValPair(zoneName, key, value) + if err != nil { + glog.Warningf("Failed to modify key value pair: %v", err) + } else { + glog.Infof("Modified key value pair for key: %v", key) + } +} + +// DeleteKeyValStateFiles deletes the state files in the /etc/nginx/state_files folder for the given virtual server. +func (lm *LocalManager) DeleteKeyValStateFiles(virtualServerName string) { + files, err := os.ReadDir(lm.stateFilesPath) + if err != nil { + glog.Warningf("Failed to read the state files directory %s: %v", lm.stateFilesPath, err) + } + for _, file := range files { + if strings.HasPrefix(file.Name(), virtualServerName+"_keyval_zone_split_clients") { + if err := os.Remove(path.Join(lm.stateFilesPath, file.Name())); err != nil { + glog.Warningf("Failed to delete the state file %s: %v", file.Name(), err) + } + } + } +} diff --git a/tests/data/common/app/weight-changes-dynamic-reload-vsr-many-splits/app.yaml b/tests/data/common/app/weight-changes-dynamic-reload-vsr-many-splits/app.yaml new file mode 100644 index 0000000000..7d8fa3200f --- /dev/null +++ b/tests/data/common/app/weight-changes-dynamic-reload-vsr-many-splits/app.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend1 +spec: + replicas: 2 + selector: + matchLabels: + app: backend1 + template: + metadata: + labels: + app: backend1 + spec: + containers: + - name: backend1 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend1-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend2 +spec: + replicas: 1 + selector: + matchLabels: + app: backend2 + template: + metadata: + labels: + app: backend2 + spec: + containers: + - name: backend2 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend2-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend3 +spec: + replicas: 1 + selector: + matchLabels: + app: backend3 + template: + metadata: + labels: + app: backend3 + spec: + containers: + - name: backend3 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend3-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend3 +--- diff --git a/tests/data/common/app/weight-changes-dynamic-reload-vsr/app.yaml b/tests/data/common/app/weight-changes-dynamic-reload-vsr/app.yaml new file mode 100644 index 0000000000..7d8fa3200f --- /dev/null +++ b/tests/data/common/app/weight-changes-dynamic-reload-vsr/app.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend1 +spec: + replicas: 2 + selector: + matchLabels: + app: backend1 + template: + metadata: + labels: + app: backend1 + spec: + containers: + - name: backend1 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend1-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend2 +spec: + replicas: 1 + selector: + matchLabels: + app: backend2 + template: + metadata: + labels: + app: backend2 + spec: + containers: + - name: backend2 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend2-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend3 +spec: + replicas: 1 + selector: + matchLabels: + app: backend3 + template: + metadata: + labels: + app: backend3 + spec: + containers: + - name: backend3 + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend3-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend3 +--- diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/configmap/nginx-config.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/configmap/nginx-config.yaml new file mode 100644 index 0000000000..bd644b7164 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/configmap/nginx-config.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + map-hash-bucket-size: "512" + map-hash-max-size: "8192" + variables-hash-bucket-size: "256" + variables-hash-max-size: "16384" diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server-many.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server-many.yaml new file mode 100644 index 0000000000..71283e8a86 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server-many.yaml @@ -0,0 +1,9 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server-route-many +spec: + host: virtual-server-route.example.com + routes: + - path: "/backends" + route: backends2-namespace/backendsmany diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server.yaml new file mode 100644 index 0000000000..b6162714c5 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/standard/virtual-server.yaml @@ -0,0 +1,9 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server-route +spec: + host: virtual-server-route.example.com + routes: + - path: "/backends" + route: backends-namespace/backends diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-initial.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-initial.yaml new file mode 100644 index 0000000000..5fde75a060 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-initial.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backends +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend2 + service: backend2-svc + port: 80 + subroutes: + - path: "/backends/backends" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-initial.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-initial.yaml new file mode 100644 index 0000000000..2ee657fc89 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-initial.yaml @@ -0,0 +1,270 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backendsmany +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend2 + service: backend2-svc + port: 80 + subroutes: + - path: "/backends/backends1" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends2" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends3" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends4" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends5" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends6" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends7" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends8" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends9" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends10" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends11" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends12" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends13" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends14" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends15" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends16" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends17" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends18" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends19" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends20" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends21" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends22" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends23" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends24" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends25" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends26" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends27" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends28" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends29" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends30" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends31" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends32" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-swap.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-swap.yaml new file mode 100644 index 0000000000..21a3f92ab8 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-swap.yaml @@ -0,0 +1,270 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backendsmany +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend2 + service: backend2-svc + port: 80 + subroutes: # /backends/backends32 is swapped + - path: "/backends/backends1" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends2" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends3" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends4" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends5" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends6" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends7" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends8" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends9" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends10" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends11" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends12" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends13" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends14" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends15" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends16" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends17" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends18" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends19" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends20" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends21" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends22" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends23" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends24" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends25" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends26" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends27" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends28" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends29" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends30" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends31" + splits: + - weight: 100 + action: + pass: backend1 + - weight: 0 + action: + pass: backend2 + - path: "/backends/backends32" + splits: + - weight: 0 + action: + pass: backend1 + - weight: 100 + action: + pass: backend2 diff --git a/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-swap.yaml b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-swap.yaml new file mode 100644 index 0000000000..449c60c692 --- /dev/null +++ b/tests/data/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-swap.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: backends +spec: + host: virtual-server-route.example.com + upstreams: + - name: backend1 + service: backend1-svc + port: 80 + - name: backend2 + service: backend2-svc + port: 80 + subroutes: + - path: "/backends/backends" + splits: + - weight: 0 + action: + pass: backend1 + - weight: 100 + action: + pass: backend2 diff --git a/tests/data/virtual-server-weight-changes-dynamic-reload/standard/virtual-server.yaml b/tests/data/virtual-server-weight-changes-dynamic-reload/standard/virtual-server.yaml new file mode 100644 index 0000000000..0b0b64c38d --- /dev/null +++ b/tests/data/virtual-server-weight-changes-dynamic-reload/standard/virtual-server.yaml @@ -0,0 +1,28 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server-splitted +spec: + host: virtual-server-splitted.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1-v1 + service: backend1-svc-v1 + port: 80 + - name: backend1-v2 + service: backend1-svc-v2 + port: 80 + routes: + - path: "/backends" + splits: + - weight: 100 + action: + pass: backend1-v1 + - weight: 0 + action: + pass: backend1-v2 + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/data/virtual-server-weight-changes-dynamic-reload/virtual-server-weight-swap.yaml b/tests/data/virtual-server-weight-changes-dynamic-reload/virtual-server-weight-swap.yaml new file mode 100644 index 0000000000..f323eb6454 --- /dev/null +++ b/tests/data/virtual-server-weight-changes-dynamic-reload/virtual-server-weight-swap.yaml @@ -0,0 +1,28 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server-splitted +spec: + host: virtual-server-splitted.example.com + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1-v1 + service: backend1-svc-v1 + port: 80 + - name: backend1-v2 + service: backend1-svc-v2 + port: 80 + routes: + - path: "/backends" + splits: + - weight: 0 + action: + pass: backend1-v1 + - weight: 100 + action: + pass: backend1-v2 + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/suite/test_v_s_route_weight_changes_dynamic_reload.py b/tests/suite/test_v_s_route_weight_changes_dynamic_reload.py new file mode 100644 index 0000000000..3b5d3d3cf0 --- /dev/null +++ b/tests/suite/test_v_s_route_weight_changes_dynamic_reload.py @@ -0,0 +1,186 @@ +import pytest +import requests +import yaml +from settings import TEST_DATA +from suite.fixtures.custom_resource_fixtures import VirtualServerRoute +from suite.utils.resources_utils import ( + create_example_app, + create_namespace_with_name_from_yaml, + delete_namespace, + ensure_response_from_backend, + get_reload_count, + wait_before_test, + wait_until_all_pods_are_ready, +) +from suite.utils.yaml_utils import get_first_host_from_yaml, get_paths_from_vsr_yaml, get_route_namespace_from_vs_yaml +from yaml.loader import Loader + +from tests.suite.utils.custom_assertions import wait_and_assert_status_code +from tests.suite.utils.vs_vsr_resources_utils import ( + create_v_s_route_from_yaml, + create_virtual_server_from_yaml, + patch_v_s_route_from_yaml, +) + + +class VSRWeightChangesDynamicReloadSetup: + """ + Encapsulate weight changes without reload details. + + Attributes: + namespace (str): + vs_host (str): + vs_name (str): + route (VirtualServerRoute): + backends_url (str): backend url + """ + + def __init__(self, namespace, vs_host, vs_name, route: VirtualServerRoute, backends_url, metrics_url): + self.namespace = namespace + self.vs_host = vs_host + self.vs_name = vs_name + self.route = route + self.backends_url = backends_url + self.metrics_url = metrics_url + + +@pytest.fixture(scope="class") +def vsr_weight_changes_dynamic_reload_setup( + request, kube_apis, ingress_controller_prerequisites, ingress_controller_endpoint +) -> VSRWeightChangesDynamicReloadSetup: + """ + Prepare an example app for weight changes without reload VSR. + + Single namespace with VS+VSR and weight changes without reload app. + + :param request: internal pytest fixture + :param kube_apis: client apis + :param ingress_controller_endpoint: + :param ingress_controller_prerequisites: + :return: + """ + + metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" + vs_routes_ns = get_route_namespace_from_vs_yaml( + f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml" + ) + ns_1 = create_namespace_with_name_from_yaml(kube_apis.v1, vs_routes_ns[0], f"{TEST_DATA}/common/ns.yaml") + print("------------------------- Deploy Virtual Server -----------------------------------") + vs_name = create_virtual_server_from_yaml( + kube_apis.custom_objects, f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml", ns_1 + ) + vs_host = get_first_host_from_yaml(f"{TEST_DATA}/{request.param['example']}/standard/virtual-server.yaml") + + print("------------------------- Deploy Virtual Server Route -----------------------------------") + vsr_name = create_v_s_route_from_yaml( + kube_apis.custom_objects, f"{TEST_DATA}/{request.param['example']}/virtual-server-route-initial.yaml", ns_1 + ) + vsr_paths = get_paths_from_vsr_yaml(f"{TEST_DATA}/{request.param['example']}/virtual-server-route-initial.yaml") + route = VirtualServerRoute(ns_1, vsr_name, vsr_paths) + backends_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.port}{vsr_paths[0]}" + + print("---------------------- Deploy weight changes without reload vsr app ----------------------------") + create_example_app(kube_apis, "weight-changes-dynamic-reload-vsr", ns_1) + wait_until_all_pods_are_ready(kube_apis.v1, ns_1) + + def fin(): + if request.config.getoption("--skip-fixture-teardown") == "no": + print("Delete test namespace") + delete_namespace(kube_apis.v1, ns_1) + + request.addfinalizer(fin) + + return VSRWeightChangesDynamicReloadSetup(ns_1, vs_host, vs_name, route, backends_url, metrics_url) + + +@pytest.mark.vsr +@pytest.mark.smok +@pytest.mark.skip_for_nginx_oss +@pytest.mark.parametrize( + "crd_ingress_controller,vsr_weight_changes_dynamic_reload_setup, expect_reload", + [ + ( + { + "type": "complete", + "extra_args": [ + "-enable-custom-resources", + "-enable-prometheus-metrics", + "-weight-changes-dynamic-reload=true", + ], + }, + {"example": "virtual-server-route-weight-changes-dynamic-reload"}, + False, + ), + ( + { + "type": "complete", + "extra_args": [ + "-enable-custom-resources", + "-enable-prometheus-metrics", + "-weight-changes-dynamic-reload=false", + ], + }, + {"example": "virtual-server-route-weight-changes-dynamic-reload"}, + True, + ), + ], + indirect=["crd_ingress_controller", "vsr_weight_changes_dynamic_reload_setup"], + ids=["WithoutReload", "WithReload"], +) +class TestVSRWeightChangesWithReloadCondition: + + def test_vsr_weight_changes_reload_behavior( + self, kube_apis, crd_ingress_controller, vsr_weight_changes_dynamic_reload_setup, expect_reload + ): + swap_weights_config = ( + f"{TEST_DATA}/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-swap.yaml" + ) + + print("Step 1: Get a response from the backend.") + wait_and_assert_status_code( + 200, vsr_weight_changes_dynamic_reload_setup.backends_url, vsr_weight_changes_dynamic_reload_setup.vs_host + ) + resp = requests.get( + vsr_weight_changes_dynamic_reload_setup.backends_url, + headers={"host": vsr_weight_changes_dynamic_reload_setup.vs_host}, + ) + assert "backend1" in resp.text + + print("Step 2: Record the initial number of reloads.") + count_before = get_reload_count(vsr_weight_changes_dynamic_reload_setup.metrics_url) + print(f"Reload count before: {count_before}") + + print("Step 3: Apply a configuration that swaps the weights (0 100) to (100 0).") + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + vsr_weight_changes_dynamic_reload_setup.route.name, + swap_weights_config, + vsr_weight_changes_dynamic_reload_setup.route.namespace, + ) + + wait_before_test(5) + + print("Step 4: Verify hitting the other backend.") + ensure_response_from_backend( + vsr_weight_changes_dynamic_reload_setup.backends_url, vsr_weight_changes_dynamic_reload_setup.vs_host + ) + wait_and_assert_status_code( + 200, vsr_weight_changes_dynamic_reload_setup.backends_url, vsr_weight_changes_dynamic_reload_setup.vs_host + ) + resp = requests.get( + vsr_weight_changes_dynamic_reload_setup.backends_url, + headers={"host": vsr_weight_changes_dynamic_reload_setup.vs_host}, + ) + assert "backend2" in resp.text + + print("Step 5: Verify reload behavior based on the weight-changes-dynamic-reload flag.") + count_after = get_reload_count(vsr_weight_changes_dynamic_reload_setup.metrics_url) + print(f"Reload count after: {count_after}") + if expect_reload: + assert ( + count_before < count_after + ), "The reload count should increase when weights are swapped and weight-changes-dynamic-reload=false." + else: + assert ( + count_before == count_after + ), "The reload count should not change when weights are swapped and weight-changes-dynamic-reload=true." diff --git a/tests/suite/test_v_s_route_weight_changes_dynamic_reload_many_splits.py b/tests/suite/test_v_s_route_weight_changes_dynamic_reload_many_splits.py new file mode 100644 index 0000000000..8a62f247e7 --- /dev/null +++ b/tests/suite/test_v_s_route_weight_changes_dynamic_reload_many_splits.py @@ -0,0 +1,182 @@ +import pytest +import requests +import yaml +from settings import TEST_DATA +from suite.fixtures.custom_resource_fixtures import VirtualServerRoute +from suite.utils.resources_utils import ( + create_example_app, + create_namespace_with_name_from_yaml, + delete_namespace, + ensure_response_from_backend, + get_reload_count, + replace_configmap, + replace_configmap_from_yaml, + wait_before_test, + wait_until_all_pods_are_ready, +) +from suite.utils.yaml_utils import get_first_host_from_yaml, get_paths_from_vsr_yaml, get_route_namespace_from_vs_yaml +from yaml.loader import Loader + +from tests.suite.utils.custom_assertions import wait_and_assert_status_code +from tests.suite.utils.vs_vsr_resources_utils import ( + create_v_s_route_from_yaml, + create_virtual_server_from_yaml, + patch_v_s_route_from_yaml, +) + + +class VSRWeightChangesDynamicReloadManySplitsSetup: + """ + Encapsulate weight changes without reload details. + + Attributes: + namespace (str): + vs_host (str): + vs_name (str): + route (VirtualServerRoute): + backends_url (str): backend url + """ + + def __init__(self, namespace, vs_host, vs_name, route: VirtualServerRoute, backends_url, metrics_url): + self.namespace = namespace + self.vs_host = vs_host + self.vs_name = vs_name + self.route = route + self.backends_url = backends_url + self.metrics_url = metrics_url + + +@pytest.fixture(scope="class") +def vsr_weight_changes_dynamic_reload_many_splits_setup( + request, kube_apis, ingress_controller_prerequisites, ingress_controller_endpoint +) -> VSRWeightChangesDynamicReloadManySplitsSetup: + """ + Prepare an example app for weight changes without reload VSR. + + Single namespace with VS+VSR and weight changes without reload app. + + :param request: internal pytest fixture + :param kube_apis: client apis + :param ingress_controller_endpoint: + :param ingress_controller_prerequisites: + :return: + """ + + metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" + vs_routes_ns = get_route_namespace_from_vs_yaml( + f"{TEST_DATA}/{request.param['example']}/standard/virtual-server-many.yaml" + ) + ns_1 = create_namespace_with_name_from_yaml(kube_apis.v1, vs_routes_ns[0], f"{TEST_DATA}/common/ns.yaml") + print("------------------------- Deploy Virtual Server -----------------------------------") + vs_name = create_virtual_server_from_yaml( + kube_apis.custom_objects, f"{TEST_DATA}/{request.param['example']}/standard/virtual-server-many.yaml", ns_1 + ) + vs_host = get_first_host_from_yaml(f"{TEST_DATA}/{request.param['example']}/standard/virtual-server-many.yaml") + + print("------------------------- Deploy Virtual Server Route -----------------------------------") + vsr_name = create_v_s_route_from_yaml( + kube_apis.custom_objects, + f"{TEST_DATA}/{request.param['example']}/virtual-server-route-many-splits-initial.yaml", + ns_1, + ) + vsr_paths = get_paths_from_vsr_yaml( + f"{TEST_DATA}/{request.param['example']}/virtual-server-route-many-splits-initial.yaml" + ) + route = VirtualServerRoute(ns_1, vsr_name, vsr_paths) + backends_url = ( + f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.port}{vsr_paths[0][:-1]}" + ) + + print("-----------------------Apply Config Map---------------------------------------------------") + config_map_name = ingress_controller_prerequisites.config_map["metadata"]["name"] + replace_configmap_from_yaml( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/{request.param['example']}/configmap/nginx-config.yaml", + ) + + print("---------------------- Deploy weight changes dynamic reload vsr app ----------------------------") + create_example_app(kube_apis, "weight-changes-dynamic-reload-vsr-many-splits", ns_1) + wait_until_all_pods_are_ready(kube_apis.v1, ns_1) + + def fin(): + if request.config.getoption("--skip-fixture-teardown") == "no": + print("Delete test namespace") + delete_namespace(kube_apis.v1, ns_1) + replace_configmap( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + ingress_controller_prerequisites.config_map, + ) + + request.addfinalizer(fin) + + return VSRWeightChangesDynamicReloadManySplitsSetup(ns_1, vs_host, vs_name, route, backends_url, metrics_url) + + +@pytest.mark.vsr +@pytest.mark.smok +@pytest.mark.skip_for_nginx_oss +@pytest.mark.parametrize( + "crd_ingress_controller,vsr_weight_changes_dynamic_reload_many_splits_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + "-enable-custom-resources", + "-enable-prometheus-metrics", + "-weight-changes-dynamic-reload=true", + "-v=3", + ], + }, + {"example": "virtual-server-route-weight-changes-dynamic-reload"}, + ), + ], + indirect=["crd_ingress_controller", "vsr_weight_changes_dynamic_reload_many_splits_setup"], +) +class TestVSRWeightChangesDynamicReloadManySplits: + + def test_vsr_weight_changes_dynamic_reload_many_splits( + self, kube_apis, crd_ingress_controller, vsr_weight_changes_dynamic_reload_many_splits_setup + ) -> None: + """ + This test checks if 32 splits can be created when the following values are specified in the configmap + map-hash-bucket-size: "512" + map-hash-max-size: "8192" + variables-hash-bucket-size: "256" + variables-hash-max-size: "16384" + + and also that weight-changes-dynamic-reload is set to true + """ + swap_weights_config = ( + f"{TEST_DATA}/virtual-server-route-weight-changes-dynamic-reload/virtual-server-route-many-splits-swap.yaml" + ) + + print("Step 1: Get a response from the backend.") + backends32_url = f"{vsr_weight_changes_dynamic_reload_many_splits_setup.backends_url}32" + wait_and_assert_status_code(200, backends32_url, vsr_weight_changes_dynamic_reload_many_splits_setup.vs_host) + resp = requests.get( + backends32_url, + headers={"host": vsr_weight_changes_dynamic_reload_many_splits_setup.vs_host}, + ) + assert "backend1" in resp.text + + print("Step 2: Apply a configuration that swaps the weights (0 100) to (100 0).") + patch_v_s_route_from_yaml( + kube_apis.custom_objects, + vsr_weight_changes_dynamic_reload_many_splits_setup.route.name, + swap_weights_config, + vsr_weight_changes_dynamic_reload_many_splits_setup.route.namespace, + ) + + print("Step 3: Verify hitting the other backend.") + ensure_response_from_backend(backends32_url, vsr_weight_changes_dynamic_reload_many_splits_setup.vs_host) + wait_and_assert_status_code(200, backends32_url, vsr_weight_changes_dynamic_reload_many_splits_setup.vs_host) + resp = requests.get( + backends32_url, + headers={"host": vsr_weight_changes_dynamic_reload_many_splits_setup.vs_host}, + ) + assert "backend2" in resp.text diff --git a/tests/suite/test_virtual_server_weight_changes_without_reload.py b/tests/suite/test_virtual_server_weight_changes_without_reload.py new file mode 100644 index 0000000000..a7a84f7f78 --- /dev/null +++ b/tests/suite/test_virtual_server_weight_changes_without_reload.py @@ -0,0 +1,101 @@ +import pytest +import requests +import yaml +from settings import TEST_DATA +from suite.utils.custom_assertions import wait_and_assert_status_code +from suite.utils.resources_utils import ensure_response_from_backend, get_reload_count, wait_before_test +from suite.utils.vs_vsr_resources_utils import patch_virtual_server_from_yaml + + +@pytest.mark.vs +@pytest.mark.smok +@pytest.mark.skip_for_nginx_oss +@pytest.mark.parametrize( + "crd_ingress_controller, virtual_server_setup, expect_reload", + [ + ( + { + "type": "complete", + "extra_args": [ + "-enable-custom-resources", + "-enable-prometheus-metrics", + "-weight-changes-dynamic-reload=true", + ], + }, + {"example": "virtual-server-weight-changes-dynamic-reload", "app_type": "split"}, + False, + ), + ( + { + "type": "complete", + "extra_args": [ + "-enable-custom-resources", + "-enable-prometheus-metrics", + "-weight-changes-dynamic-reload=false", + ], + }, + { + "example": "virtual-server-weight-changes-dynamic-reload", + "app_type": "split", + }, + True, + ), + ], + indirect=["crd_ingress_controller", "virtual_server_setup"], + ids=[ + "WithoutReload", + "WithReload", + ], +) +class TestWeightChangesWithReloadCondition: + def test_weight_changes_reload_behavior( + self, kube_apis, crd_ingress_controller, virtual_server_setup, expect_reload + ): + initial_weights_config = ( + f"{TEST_DATA}/virtual-server-weight-changes-dynamic-reload/standard/virtual-server.yaml" + ) + patch_virtual_server_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + initial_weights_config, + virtual_server_setup.namespace, + ) + swap_weights_config = ( + f"{TEST_DATA}/virtual-server-weight-changes-dynamic-reload/virtual-server-weight-swap.yaml" + ) + + print("Step 1: Get a response from the backend.") + wait_and_assert_status_code(200, virtual_server_setup.backend_1_url, virtual_server_setup.vs_host) + resp = requests.get(virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host}) + assert "backend1-v1" in resp.text + + print("Step 2: Record the initial number of reloads.") + count_before = get_reload_count(virtual_server_setup.metrics_url) + print(f"Reload count before: {count_before}") + + print("Step 3: Apply a configuration that swaps the weights (0 100) to (100 0).") + patch_virtual_server_from_yaml( + kube_apis.custom_objects, virtual_server_setup.vs_name, swap_weights_config, virtual_server_setup.namespace + ) + + print("Wait after applying config") + wait_before_test(5) + + print("Step 4: Verify hitting the other backend.") + ensure_response_from_backend(virtual_server_setup.backend_1_url, virtual_server_setup.vs_host) + wait_and_assert_status_code(200, virtual_server_setup.backend_1_url, virtual_server_setup.vs_host) + resp = requests.get(virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host}) + assert "backend1-v2" in resp.text + + print("Step 5: Verify reload behavior based on the weight-changes-dynamic-reload flag.") + count_after = get_reload_count(virtual_server_setup.metrics_url) + print(f"Reload count after: {count_after}") + + if expect_reload: + assert ( + count_before < count_after + ), "The reload count should increase when weights are swapped and weight-changes-dynamic-reload=false." + else: + assert ( + count_before == count_after + ), "The reload count should not change when weights are swapped and weight-changes-dynamic-reload=true."