8000 [Feature] Add operator shutdown handler for graceful termination (#965) · sharekey/kube-arangodb@cf46f5b · GitHub
[go: up one dir, main page]

Skip to content

Commit cf46f5b

Browse files
[Feature] Add operator shutdown handler for graceful termination (arangodb#965)
1 parent 5499e63 commit cf46f5b

File tree

5 files changed

+66
-6
lines changed

5 files changed

+66
-6
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
- (Feature) (ACS) Add Resource plan
88
- (Feature) Allow raw json value for license token-v2
99
- (Update) Replace `beta.kubernetes.io/arch` to `kubernetes.io/arch` in Operator Chart
10+
- (Feature) Add operator shutdown handler for graceful termination
1011

1112
## [1.2.12](https://github.com/arangodb/kube-arangodb/tree/1.2.12) (2022-05-10)
1213
- (Feature) Add CoreV1 Endpoints Inspector

cmd/main.go

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,10 @@ import (
3333

3434
"github.com/arangodb/kube-arangodb/pkg/util/globals"
3535

36-
operatorHTTP "github.com/arangodb/kube-arangodb/pkg/util/http"
3736
"github.com/gin-gonic/gin"
3837

38+
operatorHTTP "github.com/arangodb/kube-arangodb/pkg/util/http"
39+
3940
"github.com/arangodb/kube-arangodb/pkg/version"
4041

4142
"github.com/arangodb/kube-arangodb/pkg/operator/scope"
@@ -61,6 +62,9 @@ import (
6162
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
6263
"k8s.io/client-go/tools/record"
6364

65+
v1 "k8s.io/api/core/v1"
66+
"k8s.io/klog"
67+
6468
"github.com/arangodb/kube-arangodb/pkg/crd"
6569
"github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme"
6670
"github.com/arangodb/kube-arangodb/pkg/logging"
@@ -71,8 +75,6 @@ import (
7175
"github.com/arangodb/kube-arangodb/pkg/util/kclient"
7276
"github.com/arangodb/kube-arangodb/pkg/util/probe"
7377
"github.com/arangodb/kube-arangodb/pkg/util/retry"
74-
v1 "k8s.io/api/core/v1"
75-
"k8s.io/klog"
7678
)
7779

7880
const (
@@ -83,6 +85,8 @@ const (
8385
defaultAlpineImage = "alpine:3.7"
8486
defaultMetricsExporterImage = "arangodb/arangodb-exporter:0.1.6"
8587
defaultArangoImage = "arangodb/arangodb:latest"
88+
defaultShutdownDelay = 2 * time.Second
89+
defaultShutdownTimeout = 30 * time.Second
8690

8791
UBIImageEnv util.EnvironmentVariable = "RELATED_IMAGE_UBI"
8892
ArangoImageEnv util.EnvironmentVariable = "RELATED_IMAGE_DATABASE"
@@ -121,6 +125,10 @@ var (
121125
singleMode bool
122126
scope string
123127
}
128+
shutdownOptions struct {
129+
delay time.Duration
130+
timeout time.Duration
131+
}
124132
crdOptions struct {
125133
install bool
126134
}
@@ -180,6 +188,8 @@ func init() {
180188
f.DurationVar(&operatorTimeouts.arangoDCheck, "timeout.arangod-check", globals.DefaultArangoDCheckTimeout, "The version check request timeout to the ArangoDB")
181189
f.DurationVar(&operatorTimeouts.agency, "timeout.agency", globals.DefaultArangoDAgencyTimeout, "The Agency read timeout")
182190
f.DurationVar(&operatorTimeouts.reconciliation, "timeout.reconciliation", globals.DefaultReconciliationTimeout, "The reconciliation timeout to the ArangoDB CR")
191+
f.DurationVar(&shutdownOptions.delay, "shutdown.delay", defaultShutdownDelay, "The delay before running shutdown handlers")
192+
f.DurationVar(&shutdownOptions.timeout, "shutdown.timeout", defaultShutdownTimeout, "Timeout for shutdown handlers")
183193
f.BoolVar(&operatorOptions.scalingIntegrationEnabled, "internal.scaling-integration", true, "Enable Scaling Integration")
184194
f.Int64Var(&operatorKubernetesOptions.maxBatchSize, "kubernetes.max-batch-size", globals.DefaultKubernetesRequestBatchSize, "Size of batch during objects read")
185195
f.Float32Var(&operatorKubernetesOptions.qps, "kubernetes.qps", kclient.DefaultQPS, "Number of queries per second for k8s API")
@@ -421,6 +431,8 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
421431
ArangoImage: operatorOptions.arangoImage,
422432
SingleMode: operatorOptions.singleMode,
423433
Scope: scope,
434+
ShutdownDelay: shutdownOptions.delay,
435+
ShutdownTimeout: shutdownOptions.timeout,
424436
}
425437
deps := operator.Dependencies{
426438
LogService: logService,

docs/design/README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@
99
- [Status](./status.md)
1010
- [Upgrading](./upgrading.md)
1111
- [Rotating Pods](./rotating.md)
12-
- [Maintenance](./maintenance.md)
12+
- [Maintenance](./maintenance.md)
13+
- [Additional configuration](./additional_configuration.md)
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Additional configuration
2+
3+
It is possible to additionally fine-tune operator behavior by
4+
providing arguments via `operator.args` chart template value.
5+
6+
For example, you can specify burst size for k8s API requests or how long the operator
7+
should wait for ArangoDeployment termination after receiving interruption signal:
8+
```
9+
operator:
10+
args: ["--kubernetes.burst=40", --shutdown.timeout=2m"]
11+
```
12+
13+
The full list of available arguments can be retrieved using
14+
```
15+
export OPERATOR_IMAGE=arangodb/kube-arangodb:1.2.9
16+
kubectl run arango-operator-help --image=$OPERATOR_IMAGE -i --rm --restart=Never -- --help
17+
```

pkg/operator/operator.go

Lines changed: 31 additions & 2 deletions
< EEDB /tr>
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ import (
5454
"github.com/arangodb/kube-arangodb/pkg/operatorV2/event"
5555
"github.com/arangodb/kube-arangodb/pkg/replication"
5656
"github.com/arangodb/kube-arangodb/pkg/storage"
57+
"github.com/arangodb/kube-arangodb/pkg/util"
5758
"github.com/arangodb/kube-arangodb/pkg/util/constants"
5859
"github.com/arangodb/kube-arangodb/pkg/util/kclient"
5960
"github.com/arangodb/kube-arangodb/pkg/util/probe"
@@ -105,6 +106,8 @@ type Config struct {
105106
ScalingIntegrationEnabled bool
106107
SingleMode bool
107108
Scope scope.Scope
109+
ShutdownDelay time.Duration
110+
ShutdownTimeout time.Duration
108111
}
109112

110113
type Dependencies struct {
@@ -177,8 +180,34 @@ func (o *Operator) Run() {
177180
go o.runWithoutLeaderElection("arango-k2k-cluster-sync-operator", constants.ClusterSyncLabelRole, o.onStartK2KClusterSync, o.Dependencies.K2KClusterSyncProbe)
178181
}
179182
}
180-
// Wait until process terminates
181-
<-context.TODO().Done()
183+
184+
ctx := util.CreateSignalContext(context.Background())
185+
<-ctx.Done()
186+
o.log.Info().Msgf("Got interrupt signal, running shutdown handler in %s...", o.Config.ShutdownDelay)
187+
time.Sleep(o.Config.ShutdownDelay)
188+
o.handleShutdown()
189+
}
190+
191+
func (o *Operator) handleShutdown() {
192+
o.log.Info().Msg("Waiting for deployments termination...")
193+
shutdownCh := make(chan struct{})
194+
go func() {
195+
for {
196+
if len(o.deployments) == 0 {
197+
break
198+
}
199+
time.Sleep(time.Second)
200+
}
201+
shutdownCh <- struct{}{}
202+
}()
203+
select {
204+
case <-shutdownCh:
205+
o.log.Info().Msg("All deployments terminated, exiting.")
206+
return
207+
case <-time.After(o.Config.ShutdownTimeout):
208+
o.log.Info().Msg("Timeout reached before all deployments terminated, exiting.")
209+
return
210+
}
182211
}
183212

184213
// onStartDeployment starts the deployment operator and run till given channel is closed.

0 commit comments

Comments
 (0)
0