-
Notifications
You must be signed in to change notification settings - Fork 69
Expand file tree
/
Copy pathdebug_kube.go
More file actions
321 lines (270 loc) · 9.37 KB
/
debug_kube.go
File metadata and controls
321 lines (270 loc) · 9.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
package main
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"runtime"
"github.com/sourcegraph/sourcegraph/lib/errors"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
"github.com/sourcegraph/src-cli/internal/exec"
)
func init() {
usage := `
'src debug kube' invokes kubectl diagnostic commands targeting kubectl's current-context, writing returns to an archive.
Usage:
src debug kube [command options]
Flags:
-o Specify the name of the output zip archive.
-n Specify the namespace passed to kubectl commands. If not specified the 'default' namespace is used.
--no-config Don't include Sourcegraph configuration json.
Examples:
$ src debug kube -o debug.zip
$ src -v debug kube -n ns-sourcegraph -o foo
$ src debug kube -no-configs -o bar.zip
`
flagSet := flag.NewFlagSet("kube", flag.ExitOnError)
var base string
var namespace string
var excludeConfigs bool
flagSet.StringVar(&base, "o", "debug.zip", "The name of the output zip archive")
flagSet.StringVar(&namespace, "n", "default", "The namespace passed to kubectl commands, if not specified the 'default' namespace is used")
flagSet.BoolVar(&excludeConfigs, "no-configs", false, "If true, exclude Sourcegraph configuration files. Defaults to false.")
handler := func(args []string) error {
if err := flagSet.Parse(args); err != nil {
return err
}
// process -o flag to get zipfile and base directory names
if base == "" {
return errors.New("empty -o flag")
}
base, baseDir := processBaseDir(base)
// init context
ctx := context.Background()
// open pipe to output file
out, err := os.OpenFile(base, os.O_CREATE|os.O_RDWR|os.O_EXCL, 0666)
if err != nil {
return errors.Wrapf(err, "failed to open file %q", base)
}
defer out.Close()
// init zip writer
zw := zip.NewWriter(out)
defer zw.Close()
// Gather data for safety check
pods, err := selectPods(ctx, namespace)
if err != nil {
return errors.Wrap(err, "failed to get pods")
}
kubectx, err := exec.CommandContext(ctx, "kubectl", "config", "current-context").CombinedOutput()
if err != nil {
return errors.Wrapf(err, "failed to get current-context")
}
// Safety check user knows what they've targeted with this command
log.Printf("Archiving kubectl data for %d pods\n SRC_ENDPOINT: %v\n Context: %s Namespace: %v\n Output filename: %v", len(pods.Items), cfg.Endpoint, kubectx, namespace, base)
if verified, _ := verify("Do you want to start writing to an archive?"); !verified {
return nil
}
err = archiveKube(ctx, zw, *verbose, !excludeConfigs, namespace, baseDir, pods)
if err != nil {
return err
}
return nil
}
debugCommands = append(debugCommands, &command{
flagSet: flagSet,
handler: handler,
usageFunc: func() {
fmt.Println(usage)
},
})
}
type podList struct {
Items []struct {
Metadata struct {
Name string
}
Spec struct {
Containers []struct {
Name string
}
}
}
}
// Runs common kubectl functions and archive results to zip file
func archiveKube(ctx context.Context, zw *zip.Writer, verbose, archiveConfigs bool, namespace, baseDir string, pods podList) error {
// Create a context with a cancel function that we call when returning
// from archiveKube. This ensures we close all pending go-routines when returning
// early because of an error.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// setup channel for slice of archive function outputs, as well as throttling semaphore
ch := make(chan *archiveFile)
g, ctx := errgroup.WithContext(ctx)
semaphore := semaphore.NewWeighted(int64(runtime.GOMAXPROCS(0)))
run := func(f func() *archiveFile) {
g.Go(func() error {
if err := semaphore.Acquire(ctx, 1); err != nil {
return err
}
defer semaphore.Release(1)
if file := f(); file != nil {
ch <- file
}
return nil
})
}
// create goroutine to get pods
run(func() *archiveFile { return getPods(ctx, namespace, baseDir) })
// create goroutine to get kubectl events
run(func() *archiveFile { return getEvents(ctx, namespace, baseDir) })
// create goroutine to get persistent volumes
run(func() *archiveFile { return getPV(ctx, namespace, baseDir) })
// create goroutine to get persistent volumes claim
run(func() *archiveFile { return getPVC(ctx, namespace, baseDir) })
// start goroutine to run kubectl logs for each pod's container's
for _, pod := range pods.Items {
for _, container := range pod.Spec.Containers {
p := pod.Metadata.Name
c := container.Name
run(func() *archiveFile { return getPodLog(ctx, p, c, namespace, baseDir) })
}
}
// start goroutine to run kubectl logs --previous for each pod's container's
// won't write to zip on err, only passes bytes to channel if err not nil
for _, pod := range pods.Items {
for _, container := range pod.Spec.Containers {
p := pod.Metadata.Name
c := container.Name
run(func() *archiveFile {
f := getPastPodLog(ctx, p, c, namespace, baseDir)
if f.err != nil {
if verbose {
fmt.Printf("Could not gather --previous pod logs for %s\n", p)
}
return nil
}
return f
})
}
}
// start goroutine for each pod to run kubectl describe pod
for _, pod := range pods.Items {
p := pod.Metadata.Name
run(func() *archiveFile { return getDescribe(ctx, p, namespace, baseDir) })
}
// start goroutine for each pod to run kubectl get pod <pod> -o yaml
for _, pod := range pods.Items {
p := pod.Metadata.Name
run(func() *archiveFile { return getManifest(ctx, p, namespace, baseDir) })
}
// start goroutine to get external service config
if archiveConfigs {
run(func() *archiveFile { return getExternalServicesConfig(ctx, baseDir) })
run(func() *archiveFile { return getSiteConfig(ctx, baseDir) })
}
// close channel when wait group goroutines have completed
go func() {
if err := g.Wait(); err != nil {
fmt.Printf("archiveKube failed to open wait group: %s\n", err)
os.Exit(1)
}
close(ch)
}()
// Read binaries from channel and write to archive on host machine
if err := writeChannelContentsToZip(zw, ch, verbose); err != nil {
return errors.Wrap(err, "failed to write archives from channel")
}
return nil
}
// Calls kubectl get pods and returns a list of pods to be processed
func selectPods(ctx context.Context, namespace string) (podList, error) {
// Declare buffer type var for kubectl pipe
var podsBuff bytes.Buffer
// Get all pod names as json
podsCmd := exec.CommandContext(
ctx,
"kubectl", "-n", namespace, "get", "pods", "-l", "deploy=sourcegraph", "-o=json",
)
podsCmd.Stdout = &podsBuff
podsCmd.Stderr = os.Stderr
err := podsCmd.Run()
if err != nil {
errors.Wrap(err, "failed to aquire pods for subcommands with err")
}
//Decode json from podList
var pods podList
if err := json.Unmarshal(podsBuff.Bytes(), &pods); err != nil {
return pods, errors.Wrap(err, "failed to unmarshall get pods json")
}
return pods, err
}
// runs archiveFileFromCommand with arg get pods
func getPods(ctx context.Context, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "getPods.txt"),
"kubectl", "-n", namespace, "get", "pods", "-o", "wide",
)
}
// runs archiveFileFromCommand with arg get events
func getEvents(ctx context.Context, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "events.txt"),
"kubectl", "-n", namespace, "get", "events",
)
}
// runs archiveFileFromCommand with arg get pv
func getPV(ctx context.Context, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "persistent-volumes.txt"),
"kubectl", "-n", namespace, "get", "pv",
)
}
// runs archiveFileFromCommand with arg get pvc
func getPVC(ctx context.Context, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "persistent-volume-claims.txt"),
"kubectl", "-n", namespace, "get", "pvc",
)
}
// runs archiveFileFromCommand with arg logs $POD -c $CONTAINER
func getPodLog(ctx context.Context, podName, containerName, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "pods", podName, fmt.Sprintf("%s.log", containerName)),
"kubectl", "-n", namespace, "logs", podName, "-c", containerName,
)
}
// runs archiveFileFromCommand with arg logs --previous $POD -c $CONTAINER
func getPastPodLog(ctx context.Context, podName, containerName, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "pods", podName, fmt.Sprintf("prev-%s.log", containerName)),
"kubectl", "-n", namespace, "logs", "--previous", podName, "-c", containerName,
)
}
// runs archiveFileFromCommand with arg describe pod $POD
func getDescribe(ctx context.Context, podName, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "pods", podName, fmt.Sprintf("describe-%s.txt", podName)),
"kubectl", "-n", namespace, "describe", "pod", podName,
)
}
// runs archiveFileFromCommand with arg get pod $POD -o yaml
func getManifest(ctx context.Context, podName, namespace, baseDir string) *archiveFile {
return archiveFileFromCommand(
ctx,
filepath.Join(baseDir, "kubectl", "pods", podName, fmt.Sprintf("manifest-%s.yaml", podName)),
"kubectl", "-n", namespace, "get", "pod", podName, "-o", "yaml",
)
}