8000 [Feature] [1.1.5] Add member shutdown option (#700) · sharekey/kube-arangodb@1e6f7df · GitHub
[go: up one dir, main page]
Skip to content

Commit 1e6f7df

Browse files
ajanikowajanikow
authored
[Feature] [1.1.5] Add member shutdown option (arangodb#700)
1 parent 12b33ae commit 1e6f7df

20 files changed

+408
-155
lines changed

‎pkg/apis/deployment/v1/deployment_status_members.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,3 +285,19 @@ func (ds DeploymentStatusMembers) MembersOfGroup(group ServerGroup) MemberStatus
285285
return MemberStatusList{}
286286
}
287287
}
288+
289+
// PodNames returns all members pod names
290+
func (ds DeploymentStatusMembers) PodNames() []string {
291+
var n []string
292+
293+
ds.ForeachServerGroup(func(group ServerGroup, list MemberStatusList) error {
294+
for _, m := range list {
295+
if m.PodName != "" {
296+
n = append(n, m.PodName)
297+
}
298+
}
299+
return nil
300+
})
301+
302+
return n
303+
}

‎pkg/apis/deployment/v1/member_phase.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,8 @@ func (p MemberPhase) IsFailed() bool {
5757
func (p MemberPhase) IsCreatedOrDrain() bool {
5858
return p == MemberPhaseCreated || p == MemberPhaseDrain
5959
}
60+
61+
// String returns string from MemberPhase
62+
func (p MemberPhase) String() string {
63+
return string(p)
64+
}

‎pkg/apis/deployment/v1/server_group_spec.go

Lines changed: 33 additions & 0 deletions
70
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,35 @@ import (
3939
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
4040
)
4141

42+
// ServerGroupShutdownMethod enum of possible shutdown methods
43+
type ServerGroupShutdownMethod string
44+
45+
// Default return default value for ServerGroupShutdownMethod
46+
func (s *ServerGroupShutdownMethod) Default() ServerGroupShutdownMethod {
47+
return ServerGroupShutdownMethodAPI
48+
}
49+
50+
// Get return current or default value of ServerGroupShutdownMethod
51+
func (s *ServerGroupShutdownMethod) Get() ServerGroupShutdownMethod {
52+
if s == nil {
53+
return s.Default()
54+
}
55+
56+
switch t := *s; t {
57+
case ServerGroupShutdownMethodAPI, ServerGroupShutdownMethodDelete:
58+
return t
59+
default:
60+
return s.Default()
61+
}
62+
}
63+
64+
const (
65+
// ServerGroupShutdownMethodAPI API Shutdown method
66+
ServerGroupShutdownMethodAPI ServerGroupShutdownMethod = "api"
67+
// ServerGroupShutdownMethodDelete Pod Delete shutdown method
68+
ServerGroupShutdownMethodDelete ServerGroupShutdownMethod = "delete"
69+
)
+
4271
// ServerGroupSpec contains the specification for all servers in a specific group (e.g. all agents)
4372
type ServerGroupSpec struct {
4473
// Count holds the requested number of servers
@@ -106,6 +135,10 @@ type ServerGroupSpec struct {
106135
ExtendedRotationCheck *bool `json:"extendedRotationCheck,omitempty"`
107136
// InitContainers Init containers specification
108137
InitContainers *ServerGroupInitContainers `json:"initContainers,omitempty"`
138+
// ShutdownMethod describe procedure of member shutdown taken by Operator
139+
ShutdownMethod *ServerGroupShutdownMethod `json:"shutdownMethod,omitempty"`
140+
// ShutdownDelay define how long operator should delay finalizer removal after shutdown
141+
ShutdownDelay *int `json:"shutdownDelay,omitempty"`
109142
}
110143

111144
// ServerGroupSpecSecurityContext contains specification for pod security context

‎pkg/apis/deployment/v1/zz_generated.deepcopy.go

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/deployment/v2alpha1/server_group_spec.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,35 @@ import (
3939
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
4040
)
4141

42+
// ServerGroupShutdownMethod enum of possible shutdown methods
43+
type ServerGroupShutdownMethod string
44+
45+
// Default return default value for ServerGroupShutdownMethod
46+
func (s *ServerGroupShutdownMethod) Default() ServerGroupShutdownMethod {
47+
return ServerGroupShutdownMethodAPI
48+
}
49+
50+
// Get return current or default value of ServerGroupShutdownMethod
51+
func (s *ServerGroupShutdownMethod) Get() ServerGroupShutdownMethod {
52+
if s == nil {
53+
return s.Default()
54+
}
55+
56+
switch t := *s; t {
57+
case ServerGroupShutdownMethodAPI, ServerGroupShutdownMethodDelete:
58+
return t
59+
default:
60+
return s.Default()
61+
}
62+
}
63+
64+
const (
65+
// ServerGroupShutdownMethodAPI API Shutdown method
66+
ServerGroupShutdownMethodAPI ServerGroupShutdownMethod = "api"
67+
// ServerGroupShutdownMethodDelete Pod Delete shutdown method
68+
ServerGroupShutdownMethodDelete ServerGroupShutdownMethod = "delete"
69+
)
70+
4271
// ServerGroupSpec contains the specification for all servers in a specific group (e.g. all agents)
4372
type ServerGroupSpec struct {
4473
// Count holds the requested number of servers
@@ -106,6 +135,8 @@ type ServerGroupSpec struct {
106135
ExtendedRotationCheck *bool `json:"extendedRotationCheck,omitempty"`
107136
// InitContainers Init containers specification
108137
InitContainers *ServerGroupInitContainers `json:"initContainers,omitempty"`
138+
// ShutdownMethod describe procedure of member shutdown taken by Operator
139+
ShutdownMethod *ServerGroupShutdownMethod `json:"shutdownMethod,omitempty"`
109140
}
110141

111142
// ServerGroupSpecSecurityContext contains specification for pod security context

‎pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/deployment/context_impl.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,11 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err
368368
return id, nil
369369
}
370370

371+
// GetPod returns pod.
372+
func (d *Deployment) GetPod(podName string) (*v1.Pod, error) {
373+
return d.deps.KubeCli.CoreV1().Pods(d.GetNamespace()).Get(context.Background(), podName, meta.GetOptions{})
374+
}
375+
371376
// DeletePod deletes a pod with given name in the namespace
372377
// of the deployment. If the pod does not exist, the error is ignored.
373378
func (d *Deployment) DeletePod(podName string) error {

‎pkg/deployment/reconcile/action.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,21 @@ import (
3232
"github.com/rs/zerolog"
3333
)
3434

35-
// Action executes a single Plan item.
36-
type Action interface {
35+
// ActionCore executes a single Plan item.
36+
type ActionCore interface {
3737
// Start performs the start of the action.
3838
// Returns true if the action is completely finished, false in case
3939
// the start time needs to be recorded and a ready condition needs to be checked.
4040
Start(ctx context.Context) (bool, error)
4141
// CheckProgress checks the progress of the action.
4242
// Returns: ready, abort, error.
4343
CheckProgress(ctx context.Context) (bool, bool, error)
44+
}
45+
46+
// Action executes a single Plan item.
47+
type Action interface {
48+
ActionCore
49+
4450
// Timeout returns the amount of time after which this action will timeout.
4551
Timeout(deploymentSpec api.DeploymentSpec) time.Duration
4652
// Return the MemberID used / created in this action

‎pkg/deployment/reconcile/action_context.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ type ActionContext interface {
7676
UpdateMember(member api.MemberStatus) error
7777
// RemoveMemberByID removes a member with given id.
7878
RemoveMemberByID(id string) error
79+
// GetPod returns pod.
80+
GetPod(podName string) (*v1.Pod, error)
7981
// DeletePod deletes a pod with given name in the namespace
8082
// of the deployment. If the pod does not exist, the error is ignored.
8183
DeletePod(podName string) error
@@ -317,6 +319,15 @@ func (ac *actionContext) RemoveMemberByID(id string) error {
317319
return nil
318320
}
319321

322+
// GetPod returns pod.
323+
func (ac *actionContext) GetPod(podName string) (*v1.Pod, error) {
324+
if pod, err := ac.context.GetPod(podName); err != nil {
325+
return nil, errors.WithStack(err)
326+
} else {
327+
return pod, nil
328+
}
329+
}
330+
320331
// DeletePod deletes a pod with given name in the namespace
321332
// of the deployment. If the pod does not exist, the error is ignored.
322333
func (ac *actionContext) DeletePod(podName string) error {

‎pkg/deployment/reconcile/action_rotate_member.go

Lines changed: 16 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ package reconcile
2525
import (
2626
"context"
2727

28+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
29+
2830
"github.com/arangodb/kube-arangodb/pkg/util/errors"
2931

3032
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
@@ -56,41 +58,17 @@ type actionRotateMember struct {
5658
// the start time needs to be recorded and a ready condition needs to be checked.
5759
func (a *actionRotateMember) Start(ctx< 426B /span> context.Context) (bool, error) {
5860
log := a.log
59-
group := a.action.Group
6061
m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
6162
if !ok {
6263
log.Error().Msg("No such member")
6364
}
64-
// Remove finalizers, so Kubernetes will quickly terminate the pod
65-
if err := a.actionCtx.RemovePodFinalizers(m.PodName); err != nil {
66-
return false, errors.WithStack(err)
67-
}
68-
if group.IsArangod() {
69-
// Invoke shutdown endpoint
70-
c, err := a.actionCtx.GetServerClient(ctx, group, a.action.MemberID)
71-
if err != nil {
72-
log.Debug().Err(err).Msg("Failed to create member client")
73-
return false, errors.WithStack(err)
74-
}
75-
removeFromCluster := false
76-
log.Debug().Bool("removeFromCluster", removeFromCluster).Msg("Shutting down member")
77-
ctx, cancel := context.WithTimeout(ctx, shutdownTimeout)
78-
defer cancel()
79-
if err := c.Shutdown(ctx, removeFromCluster); err != nil {
80-
// Shutdown failed. Let's check if we're already done
81-
if ready, _, err := a.CheckProgress(ctx); err == nil && ready {
82-
// We're done
83-
return true, nil
84-
}
85-
log.Debug().Err(err).Msg("Failed to shutdown member")
86-
return false, errors.WithStack(err)
87-
}
88-
} else if group.IsArangosync() {
89-
// Terminate pod
90-
if err := a.actionCtx.DeletePod(m.PodName); err != nil {
91-
return false, errors.WithStack(err)
92-
}
65+
66+
if ready, err := getShutdownHelper(&a.action, a.actionCtx, a.log).Start(ctx); err != nil {
67+
return false, err
68+
} else if ready {
69+
return true, nil
9370
}
71+
9472
// Update status
9573
m.Phase = api.MemberPhaseRotating
9674

@@ -110,13 +88,18 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err
11088
log.Error().Msg("No such member")
11189
return true, false, nil
11290
}
113-
if !m.Conditions.IsTrue(api.ConditionTypeTerminated) {
114-
// Pod is not yet terminated
91+
92+
if ready, abort, err := getShutdownHelper(&a.action, a.actionCtx, a.log).CheckProgress(ctx); err != nil {
93+
return false, abort, err
94+
} else if !ready {
11595
return false, false, nil
11696
}
97+
11798
// Pod is terminated, we can now remove it
11899
if err := a.actionCtx.DeletePod(m.PodName); err != nil {
119-
return false, false, errors.WithStack(err)
100+
if !k8sutil.IsNotFound(err) {
101+
return false, false, errors.WithStack(err)
102+
}
120103
}
121104
// Pod is now gone, update the member status
122105
m.Phase = api.MemberPhaseNone

0 commit comments

Comments
 (0)
0