feat(coderd/healthcheck): add health check for proxy

coder · johnstcn · Nov 24, 2023 · Nov 21, 2023 · Nov 23, 2023 · Nov 23, 2023
commit a17ec4c89666b9bc3c820f52e6a3ac6075efd39c
diff --git a/coderd/healthcheck/healthcheck.go b/coderd/healthcheck/healthcheck.go
@@ -13,17 +13,19 @@ import (
 )
 
 const (
-	SectionDERP      string = "DERP"
-	SectionAccessURL string = "AccessURL"
-	SectionWebsocket string = "Websocket"
-	SectionDatabase  string = "Database"
+	SectionDERP           string = "DERP"
+	SectionAccessURL      string = "AccessURL"
+	SectionWebsocket      string = "Websocket"
+	SectionDatabase       string = "Database"
+	SectionWorkspaceProxy string = "WorkspaceProxy"
 )
 
 type Checker interface {
 	DERP(ctx context.Context, opts *derphealth.ReportOptions) derphealth.Report
 	AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport
 	Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport
 	Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport
+	WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport
 }
 
 // @typescript-generate Report
@@ -38,20 +40,22 @@ type Report struct {
 	// FailingSections is a list of sections that have failed their healthcheck.
 	FailingSections []string `json:"failing_sections"`
 
-	DERP      derphealth.Report `json:"derp"`
-	AccessURL AccessURLReport   `json:"access_url"`
-	Websocket WebsocketReport   `json:"websocket"`
-	Database  DatabaseReport    `json:"database"`
+	DERP           derphealth.Report    `json:"derp"`
+	AccessURL      AccessURLReport      `json:"access_url"`
+	Websocket      WebsocketReport      `json:"websocket"`
+	Database       DatabaseReport       `json:"database"`
+	WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`
 
 	// The Coder version of the server that the report was generated on.
 	CoderVersion string `json:"coder_version"`
 }
 
 type ReportOptions struct {
-	AccessURL  AccessURLReportOptions
-	Database   DatabaseReportOptions
-	DerpHealth derphealth.ReportOptions
-	Websocket  WebsocketReportOptions
+	AccessURL      AccessURLReportOptions
+	Database       DatabaseReportOptions
+	DerpHealth     derphealth.ReportOptions
+	Websocket      WebsocketReportOptions
+	WorkspaceProxy WorkspaceProxyReportOptions
 
 	Checker Checker
 }
@@ -78,6 +82,11 @@ func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions)
 	return report
 }
 
+func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) {
+	report.Run(ctx, opts)
+	return report
+}
+
 func Run(ctx context.Context, opts *ReportOptions) *Report {
 	var (
 		wg     sync.WaitGroup
@@ -136,6 +145,18 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
 		report.Database = opts.Checker.Database(ctx, &opts.Database)
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer func() {
+			if err := recover(); err != nil {
+				report.WorkspaceProxy.Error = ptr.Ref(fmt.Sprint(err))
+			}
+		}()
+
+		report.WorkspaceProxy = opts.Checker.Works
8000
paceProxy(ctx, &opts.WorkspaceProxy)
+	}()
+
 	report.CoderVersion = buildinfo.Version()
 	wg.Wait()
 
@@ -153,6 +174,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
 	if !report.Database.Healthy {
 		report.FailingSections = append(report.FailingSections, SectionDatabase)
 	}
+	if !report.WorkspaceProxy.Healthy {
+		report.FailingSections = append(report.FailingSections, SectionWorkspaceProxy)
+	}
 
 	report.Healthy = len(report.FailingSections) == 0
 

diff --git a/coderd/healthcheck/healthcheck_test.go b/coderd/healthcheck/healthcheck_test.go
@@ -11,10 +11,11 @@ import (
 )
 
 type testChecker struct {
-	DERPReport      derphealth.Report
-	AccessURLReport healthcheck.AccessURLReport
-	WebsocketReport healthcheck.WebsocketReport
-	DatabaseReport  healthcheck.DatabaseReport
+	DERPReport           derphealth.Report
+	AccessURLReport      healthcheck.AccessURLReport
+	WebsocketReport      healthcheck.WebsocketReport
+	DatabaseReport       healthcheck.DatabaseReport
+	WorkspaceProxyReport healthcheck.WorkspaceProxyReport
 }
 
 func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report {
@@ -33,6 +34,10 @@ func (c *testChecker) Database(context.Context, *healthcheck.DatabaseReportOptio
 	return c.DatabaseReport
 }
 
+func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProxyReportOptions) healthcheck.WorkspaceProxyReport {
+	return c.WorkspaceProxyReport
+}
+
 func TestHealthcheck(t *testing.T) {
 	t.Parallel()
 
@@ -56,6 +61,9 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: true,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         true,
 		failingSections: []string{},
@@ -74,6 +82,9 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: true,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         false,
 		failingSections: []string{healthcheck.SectionDERP},
@@ -93,6 +104,9 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: true,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         true,
 		failingSections: []string{},
@@ -111,6 +125,9 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: true,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         false,
 		failingSections: []string{healthcheck.SectionAccessURL},
@@ -129,6 +146,9 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: true,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         false,
 		failingSections: []string{healthcheck.SectionWebsocket},
@@ -147,9 +167,33 @@ func TestHealthcheck(t *testing.T) {
 			DatabaseReport: healthcheck.DatabaseReport{
 				Healthy: false,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: true,
+			},
 		},
 		healthy:         false,
 		failingSections: []string{healthcheck.SectionDatabase},
+	}, {
+		name: "ProxyFail",
+		checker: &testChecker{
+			DERPReport: derphealth.Report{
+				Healthy: true,
+			},
+			AccessURLReport: healthcheck.AccessURLReport{
+				Healthy: true,
+			},
+			WebsocketReport: healthcheck.WebsocketReport{
+				Healthy: true,
+			},
+			DatabaseReport: healthcheck.DatabaseReport{
+				Healthy: true,
+			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy: false,
+			},
+		},
+		healthy:         false,
+		failingSections: []string{healthcheck.SectionWorkspaceProxy},
 	}, {
 		name:    "AllFail",
 		checker: &testChecker{},
@@ -159,6 +203,7 @@ func TestHealthcheck(t *testing.T) {
 			healthcheck.SectionAccessURL,
 			healthcheck.SectionWebsocket,
 			healthcheck.SectionDatabase,
+			healthcheck.SectionWorkspaceProxy,
 		},
 	}} {
 		c := c
@@ -175,6 +220,8 @@ func TestHealthcheck(t *testing.T) {
 			assert.Equal(t, c.checker.DERPReport.Warnings, report.DERP.Warnings)
 			assert.Equal(t, c.checker.AccessURLReport.Healthy, report.AccessURL.Healthy)
 			assert.Equal(t, c.checker.WebsocketReport.Healthy, report.Websocket.Healthy)
+			assert.Equal(t, c.checker.WorkspaceProxyReport.Healthy, report.WorkspaceProxy.Healthy)
+			assert.Equal(t, c.checker.WorkspaceProxyReport.Warnings, report.WorkspaceProxy.Warnings)
 			assert.NotZero(t, report.Time)
 			assert.NotZero(t, report.CoderVersion)
 		})

diff --git a/coderd/healthcheck/workspaceproxy.go b/coderd/healthcheck/workspaceproxy.go
@@ -0,0 +1,87 @@
+package healthcheck
+
+import (
+	"context"
+	"fmt"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+type WorkspaceProxyReportOptions struct {
+	// UpdateProxyHealth is a function called when healthcheck is run.
+	// This would normally be ProxyHealth.ForceUpdate().
+	// We do this because if someone mashes the healthcheck refresh button
+	// they would expect up-to-date data.
+	UpdateProxyHealth func(context.Context) error
+	// FetchWorkspaceProxies is a function that returns the available workspace proxies.
+	FetchWorkspaceProxies func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error)
+	// CurrentVersion is the current server version.
+	// We pass this in to make it easier to test.
+	CurrentVersion string
+	Logger         slog.Logger
+}
+
+// @typescript-generate Report
+type WorkspaceProxyReport struct {
+	Healthy  bool     `json:"healthy"`
+	Warnings []string `json:"warnings"`
+	Error    *string  `json:"error"`
+
+	WorkspaceProxies codersdk.RegionsResponse[codersdk.WorkspaceProxy]
+}
+
+func (r *WorkspaceProxyReport) Run(ctx context.Context, opts *WorkspaceProxyReportOptions) {
+	r.Healthy = true
+	r.Warnings = []string{}
+
+	if opts.FetchWorkspaceProxies == nil {
+		opts.Logger.Debug(ctx, "no workspace proxies configured")
+		return
+	}
+
+	if opts.UpdateProxyHealth == nil {
+		err := "opts.UpdateProxyHealth must not be nil if opts.FetchWorkspaceProxies is not nil"
+		opts.Logger.Error(ctx, "developer error: "+err)
+		r.Error = ptr.Ref(err)
+		return
+	}
+
+	if err := opts.UpdateProxyHealth(ctx); err != nil {
+		opts.Logger.Error(ctx, "failed to update proxy health: %w", err)
+		r.Error = ptr.Ref(err.Error())
+		return
+	}
+
+	proxies, err := opts.FetchWorkspaceProxies(ctx)
+	if err != nil {
+		opts.Logger.Error(ctx, "failed to fetch workspace proxies", slog.Error(err))
+		r.Healthy = false
+		r.Error = ptr.Ref(err.Error())
+		return
+	}
+
+	r.WorkspaceProxies = proxies
+
+	var numProxies int
+	var healthyProxies int
+	for _, proxy := range r.WorkspaceProxies.Regions {
+		numProxies++
+		if proxy.Healthy {
+			healthyProxies++
+		}
+
+		// check versions
+		if !buildinfo.VersionsMatch(proxy.Version, opts.CurrentVersion) {
+			opts.Logger.Warn(ctx, "proxy version mismatch",
+				slog.F("version", opts.CurrentVersion),
+				slog.F("proxy_version", proxy.Version),
+				slog.F("proxy_name", proxy.Name),
+			)
+			r.Healthy = false
+			r.Warnings = append(r.Warnings, fmt.Sprintf("Proxy %q version %q does not match primary server version %q", proxy.Name, proxy.Version, opts.CurrentVersion))
+		}
+	}
+}