coderd/coderd.go

package coderd

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/andybalholm/brotli"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/google/uuid"
	"github.com/klauspost/compress/zstd"
	"github.com/moby/moby/pkg/namesgenerator"
	"github.com/prometheus/client_golang/prometheus"
	httpSwagger "github.com/swaggo/http-swagger/v2"
	"go.opentelemetry.io/otel/trace"
	"golang.org/x/xerrors"
	"google.golang.org/api/idtoken"
	"storj.io/drpc/drpcmux"
	"storj.io/drpc/drpcserver"
	"tailscale.com/derp"
	"tailscale.com/derp/derphttp"
	"tailscale.com/tailcfg"
	"tailscale.com/types/key"
	"tailscale.com/util/singleflight"

	// Used for swagger docs.
	_ "github.com/coder/coder/v2/coderd/apidoc"
	"github.com/coder/coder/v2/coderd/externalauth"

	"cdr.dev/slog"
	"github.com/coder/coder/v2/buildinfo"
	"github.com/coder/coder/v2/cli/clibase"
	"github.com/coder/coder/v2/coderd/audit"
	"github.com/coder/coder/v2/coderd/awsidentity"
	"github.com/coder/coder/v2/coderd/batchstats"
	"github.com/coder/coder/v2/coderd/database"
	"github.com/coder/coder/v2/coderd/database/dbauthz"
	"github.com/coder/coder/v2/coderd/database/pubsub"
	"github.com/coder/coder/v2/coderd/gitsshkey"
	"github.com/coder/coder/v2/coderd/healthcheck"
	"github.com/coder/coder/v2/coderd/httpapi"
	"github.com/coder/coder/v2/coderd/httpmw"
	"github.com/coder/coder/v2/coderd/metricscache"
	"github.com/coder/coder/v2/coderd/provisionerdserver"
	"github.com/coder/coder/v2/coderd/rbac"
	"github.com/coder/coder/v2/coderd/schedule"
	"github.com/coder/coder/v2/coderd/telemetry"
	"github.com/coder/coder/v2/coderd/tracing"
	"github.com/coder/coder/v2/coderd/updatecheck"
	"github.com/coder/coder/v2/coderd/util/slice"
	"github.com/coder/coder/v2/coderd/workspaceapps"
	"github.com/coder/coder/v2/coderd/wsconncache"
	"github.com/coder/coder/v2/codersdk"
	"github.com/coder/coder/v2/codersdk/agentsdk"
	"github.com/coder/coder/v2/provisionerd/proto"
	"github.com/coder/coder/v2/provisionersdk"
	"github.com/coder/coder/v2/site"
	"github.com/coder/coder/v2/tailnet"
)

// We must only ever instantiate one httpSwagger.Handler because of a data race
// inside the handler. This issue is triggered by tests that create multiple
// coderd instances.
//
// See https://github.com/swaggo/http-swagger/issues/78
var globalHTTPSwaggerHandler http.HandlerFunc

func init() {
	globalHTTPSwaggerHandler = httpSwagger.Handler(httpSwagger.URL("/swagger/doc.json"))
}

// Options are requires parameters for Coder to start.
type Options struct {
	AccessURL *url.URL
	// AppHostname should be the wildcard hostname to use for workspace
	// applications INCLUDING the asterisk, (optional) suffix and leading dot.
	// It will use the same scheme and port number as the access URL.
	// E.g. "*.apps.coder.com" or "*-apps.coder.com".
	AppHostname string
	// AppHostnameRegex contains the regex version of options.AppHostname as
	// generated by httpapi.CompileHostnamePattern(). It MUST be set if
	// options.AppHostname is set.
	AppHostnameRegex *regexp.Regexp
	Logger           slog.Logger
	Database         database.Store
	Pubsub           pubsub.Pubsub

	// CacheDir is used for caching files served by the API.
	CacheDir string

	Auditor                        audit.Auditor
	AgentConnectionUpdateFrequency time.Duration
	AgentInactiveDisconnectTimeout time.Duration
	AWSCertificates                awsidentity.Certificates
	Authorizer                     rbac.Authorizer
	AzureCertificates              x509.VerifyOptions
	GoogleTokenValidator           *idtoken.Validator
	GithubOAuth2Config             *GithubOAuth2Config
	OIDCConfig                     *OIDCConfig
	PrometheusRegistry             *prometheus.Registry
	SecureAuthCookie               bool
	StrictTransportSecurityCfg     httpmw.HSTSConfig
	SSHKeygenAlgorithm             gitsshkey.Algorithm
	Telemetry                      telemetry.Reporter
	TracerProvider                 trace.TracerProvider
	ExternalAuthConfigs            []*externalauth.Config
	RealIPConfig                   *httpmw.RealIPConfig
	TrialGenerator                 func(ctx context.Context, email string) error
	// TLSCertificates is used to mesh DERP servers securely.
	TLSCertificates    []tls.Certificate
	TailnetCoordinator tailnet.Coordinator
	DERPServer         *derp.Server
	// BaseDERPMap is used as the base DERP map for all clients and agents.
	// Proxies are added to this list.
	BaseDERPMap                 *tailcfg.DERPMap
	DERPMapUpdateFrequency      time.Duration
	SwaggerEndpoint             bool
	SetUserGroups               func(ctx context.Context, logger slog.Logger, tx database.Store, userID uuid.UUID, groupNames []string, createMissingGroups bool) error
	SetUserSiteRoles            func(ctx context.Context, logger slog.Logger, tx database.Store, userID uuid.UUID, roles []string) error
	TemplateScheduleStore       *atomic.Pointer[schedule.TemplateScheduleStore]
	UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
	AccessControlStore          *atomic.Pointer[dbauthz.AccessControlStore]
	// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
	// workspace applications. It consists of both a signing and encryption key.
	AppSecurityKey     workspaceapps.SecurityKey
	HealthcheckFunc    func(ctx context.Context, apiKey string) *healthcheck.Report
	HealthcheckTimeout time.Duration
	HealthcheckRefresh time.Duration

	// OAuthSigningKey is the crypto key used to sign and encrypt state strings
	// related to OAuth. This is a symmetric secret key using hmac to sign payloads.
	// So this secret should **never** be exposed to the client.
	OAuthSigningKey [32]byte

	// APIRateLimit is the minutely throughput rate limit per user or ip.
	// Setting a rate limit <0 will disable the rate limiter across the entire
	// app. Some specific routes have their own configurable rate limits.
	APIRateLimit   int
	LoginRateLimit int
	FilesRateLimit int

	MetricsCacheRefreshInterval time.Duration
	AgentStatsRefreshInterval   time.Duration
	DeploymentValues            *codersdk.DeploymentValues
	// DeploymentOptions do contain the copy of DeploymentValues, and contain
	// contextual information about how the values were set.
	// Do not use DeploymentOptions to retrieve values, use DeploymentValues instead.
	// All secrets values are stripped.
	DeploymentOptions  clibase.OptionSet
	UpdateCheckOptions *updatecheck.Options // Set non-nil to enable update checking.

	// SSHConfig is the response clients use to configure config-ssh locally.
	SSHConfig codersdk.SSHConfigResponse

	HTTPClient *http.Client

	UpdateAgentMetrics func(ctx context.Context, username, workspaceName, agentName string, metrics []agentsdk.AgentMetric)
	StatsBatcher       *batchstats.Batcher

	WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
}

// @title Coder API
// @version 2.0
// @description Coderd is the service created by running coder server. It is a thin API that connects workspaces, provisioners and users. coderd stores its state in Postgres and is the only service that communicates with Postgres.
// @termsOfService https://coder.com/legal/terms-of-service

// @contact.name API Support
// @contact.url https://coder.com
// @contact.email support@coder.com

// @license.name AGPL-3.0
// @license.url https://github.com/coder/coder/blob/main/LICENSE

// @BasePath /api/v2

// @securitydefinitions.apiKey CoderSessionToken
// @in header
// @name Coder-Session-Token
// New constructs a Coder API handler.
func New(options *Options) *API {
	if options == nil {
		options = &Options{}
	}

	// Safety check: if we're not running a unit test, we *must* have a Prometheus registry.
	if options.PrometheusRegistry == nil && flag.Lookup("test.v") == nil {
		panic("developer error: options.PrometheusRegistry is nil and not running a unit test")
	}

	if options.DeploymentValues.DisableOwnerWorkspaceExec {
		rbac.ReloadBuiltinRoles(&rbac.RoleOptions{
			NoOwnerWorkspaceExec: true,
		})
	}

	if options.Authorizer == nil {
		options.Authorizer = rbac.NewCachingAuthorizer(options.PrometheusRegistry)
	}

	if options.AccessControlStore == nil {
		options.AccessControlStore = &atomic.Pointer[dbauthz.AccessControlStore]{}
		var tacs dbauthz.AccessControlStore = dbauthz.AGPLTemplateAccessControlStore{}
		options.AccessControlStore.Store(&tacs)
	}

	options.Database = dbauthz.New(
		options.Database,
		options.Authorizer,
		options.Logger.Named("authz_querier"),
		options.AccessControlStore,
	)

	experiments := ReadExperiments(
		options.Logger, options.DeploymentValues.Experiments.Value(),
	)
	if options.AppHostname != "" && options.AppHostnameRegex == nil || options.AppHostname == "" && options.AppHostnameRegex != nil {
		panic("coderd: both AppHostname and AppHostnameRegex must be set or unset")
	}
	if options.AgentConnectionUpdateFrequency == 0 {
		options.AgentConnectionUpdateFrequency = 15 * time.Second
	}
	if options.AgentInactiveDisconnectTimeout == 0 {
		// Multiply the update by two to allow for some lag-time.
		options.AgentInactiveDisconnectTimeout = options.AgentConnectionUpdateFrequency * 2
		// Set a minimum timeout to avoid disconnecting too soon.
		if options.AgentInactiveDisconnectTimeout < 2*time.Second {
			options.AgentInactiveDisconnectTimeout = 2 * time.Second
		}
	}
	if options.AgentStatsRefreshInterval == 0 {
		options.AgentStatsRefreshInterval = 5 * time.Minute
	}
	if options.MetricsCacheRefreshInterval == 0 {
		options.MetricsCacheRefreshInterval = time.Hour
	}
	if options.APIRateLimit == 0 {
		options.APIRateLimit = 512
	}
	if options.LoginRateLimit == 0 {
		options.LoginRateLimit = 60
	}
	if options.FilesRateLimit == 0 {
		options.FilesRateLimit = 12
	}
	if options.PrometheusRegistry == nil {
		options.PrometheusRegistry = prometheus.NewRegistry()
	}
	if options.DERPServer == nil {
		options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp")))
	}
	if options.DERPMapUpdateFrequency == 0 {
		options.DERPMapUpdateFrequency = 5 * time.Second
	}
	if options.TailnetCoordinator == nil {
		options.TailnetCoordinator = tailnet.NewCoordinator(options.Logger)
	}
	if options.Auditor == nil {
		options.Auditor = audit.NewNop()
	}
	if options.SSHConfig.HostnamePrefix == "" {
		options.SSHConfig.HostnamePrefix = "coder."
	}
	if options.TracerProvider == nil {
		options.TracerProvider = trace.NewNoopTracerProvider()
	}
	if options.SetUserGroups == nil {
		options.SetUserGroups = func(ctx context.Context, logger slog.Logger, _ database.Store, userID uuid.UUID, groups []string, createMissingGroups bool) error {
			logger.Warn(ctx, "attempted to assign OIDC groups without enterprise license",
				slog.F("user_id", userID), slog.F("groups", groups), slog.F("create_missing_groups", createMissingGroups),
			)
			return nil
		}
	}
	if options.SetUserSiteRoles == nil {
		options.SetUserSiteRoles = func(ctx context.Context, logger slog.Logger, _ database.Store, userID uuid.UUID, roles []string) error {
			logger.Warn(ctx, "attempted to assign OIDC user roles without enterprise license",
				slog.F("user_id", userID), slog.F("roles", roles),
			)
			return nil
		}
	}
	if options.TemplateScheduleStore == nil {
		options.TemplateScheduleStore = &atomic.Pointer[schedule.TemplateScheduleStore]{}
	}
	if options.TemplateScheduleStore.Load() == nil {
		v := schedule.NewAGPLTemplateScheduleStore()
		options.TemplateScheduleStore.Store(&v)
	}
	if options.UserQuietHoursScheduleStore == nil {
		options.UserQuietHoursScheduleStore = &atomic.Pointer[schedule.UserQuietHoursScheduleStore]{}
	}
	if options.UserQuietHoursScheduleStore.Load() == nil {
		v := schedule.NewAGPLUserQuietHoursScheduleStore()
		options.UserQuietHoursScheduleStore.Store(&v)
	}

	if options.StatsBatcher == nil {
		panic("developer error: options.StatsBatcher is nil")
	}

	siteCacheDir := options.CacheDir
	if siteCacheDir != "" {
		siteCacheDir = filepath.Join(siteCacheDir, "site")
	}
	binFS, binHashes, err := site.ExtractOrReadBinFS(siteCacheDir, site.FS())
	if err != nil {
		panic(xerrors.Errorf("read site bin failed: %w", err))
	}

	metricsCache := metricscache.New(
		options.Database,
		options.Logger.Named("metrics_cache"),
		metricscache.Intervals{
			TemplateDAUs:    options.MetricsCacheRefreshInterval,
			DeploymentStats: options.AgentStatsRefreshInterval,
		},
	)

	oauthConfigs := &httpmw.OAuth2Configs{
		Github: options.GithubOAuth2Config,
		OIDC:   options.OIDCConfig,
	}

	staticHandler := site.New(&site.Options{
		BinFS:         binFS,
		BinHashes:     binHashes,
		Database:      options.Database,
		SiteFS:        site.FS(),
		OAuth2Configs: oauthConfigs,
		DocsURL:       options.DeploymentValues.DocsURL.String(),
	})
	staticHandler.Experiments.Store(&experiments)

	ctx, cancel := context.WithCancel(context.Background())
	r := chi.NewRouter()

	// nolint:gocritic // Load deployment ID. This never changes
	depID, err := options.Database.GetDeploymentID(dbauthz.AsSystemRestricted(ctx))
	if err != nil {
		panic(xerrors.Errorf("get deployment ID: %w", err))
	}
	api := &API{
		ctx:          ctx,
		cancel:       cancel,
		DeploymentID: depID,

		ID:          uuid.New(),
		Options:     options,
		RootHandler: r,
		SiteHandler: staticHandler,
		HTTPAuth: &HTTPAuthorizer{
			Authorizer: options.Authorizer,
			Logger:     options.Logger,
		},
		WorkspaceAppsProvider: workspaceapps.NewDBTokenProvider(
			options.Logger.Named("workspaceapps"),
			options.AccessURL,
			options.Authorizer,
			options.Database,
			options.DeploymentValues,
			oauthConfigs,
			options.AgentInactiveDisconnectTimeout,
			options.AppSecurityKey,
		),
		metricsCache:                metricsCache,
		Auditor:                     atomic.Pointer[audit.Auditor]{},
		TemplateScheduleStore:       options.TemplateScheduleStore,
		UserQuietHoursScheduleStore: options.UserQuietHoursScheduleStore,
		AccessControlStore:          options.AccessControlStore,
		Experiments:                 experiments,
		healthCheckGroup:            &singleflight.Group[string, *healthcheck.Report]{},
		Acquirer: provisionerdserver.NewAcquirer(
			ctx,
			options.Logger.Named("acquirer"),
			options.Database,
			options.Pubsub),
	}
	if options.UpdateCheckOptions != nil {
		api.updateChecker = updatecheck.New(
			options.Database,
			options.Logger.Named("update_checker"),
			*options.UpdateCheckOptions,
		)
	}
	if options.HealthcheckFunc == nil {
		options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report {
			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
				DB:        options.Database,
				AccessURL: options.AccessURL,
				DERPMap:   api.DERPMap(),
				APIKey:    apiKey,
			})
		}
	}
	if options.HealthcheckTimeout == 0 {
		options.HealthcheckTimeout = 30 * time.Second
	}
	if options.HealthcheckRefresh == 0 {
		options.HealthcheckRefresh = 10 * time.Minute
	}

	var oidcAuthURLParams map[string]string
	if options.OIDCConfig != nil {
		oidcAuthURLParams = options.OIDCConfig.AuthURLParams
	}

	api.Auditor.Store(&options.Auditor)
	api.TailnetCoordinator.Store(&options.TailnetCoordinator)
	if api.Experiments.Enabled(codersdk.ExperimentSingleTailnet) {
		api.agentProvider, err = NewServerTailnet(api.ctx,
			options.Logger,
			options.DERPServer,
			api.DERPMap,
			options.DeploymentValues.DERP.Config.ForceWebSockets.Value(),
			func(context.Context) (tailnet.MultiAgentConn, error) {
				return (*api.TailnetCoordinator.Load()).ServeMultiAgent(uuid.New()), nil
			},
			wsconncache.New(api._dialWorkspaceAgentTailnet, 0),
			api.TracerProvider,
		)
		if err != nil {
			panic("failed to setup server tailnet: " + err.Error())
		}
	} else {
		api.agentProvider = &wsconncache.AgentProvider{
			Cache: wsconncache.New(api._dialWorkspaceAgentTailnet, 0),
		}
	}

	workspaceAppsLogger := options.Logger.Named("workspaceapps")
	if options.WorkspaceAppsStatsCollectorOptions.Logger == nil {
		named := workspaceAppsLogger.Named("stats_collector")
		options.WorkspaceAppsStatsCollectorOptions.Logger = &named
	}
	if options.WorkspaceAppsStatsCollectorOptions.Reporter == nil {
		options.WorkspaceAppsStatsCollectorOptions.Reporter = workspaceapps.NewStatsDBReporter(options.Database, workspaceapps.DefaultStatsDBReporterBatchSize)
	}

	api.workspaceAppServer = &workspaceapps.Server{
		Logger: workspaceAppsLogger,

		DashboardURL:  api.AccessURL,
		AccessURL:     api.AccessURL,
		Hostname:      api.AppHostname,
		HostnameRegex: api.AppHostnameRegex,
		RealIPConfig:  options.RealIPConfig,

		SignedTokenProvider: api.WorkspaceAppsProvider,
		AgentProvider:       api.agentProvider,
		AppSecurityKey:      options.AppSecurityKey,
		StatsCollector:      workspaceapps.NewStatsCollector(options.WorkspaceAppsStatsCollectorOptions),

		DisablePathApps:  options.DeploymentValues.DisablePathApps.Value(),
		SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(),
	}

	apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
		DB:                          options.Database,
		OAuth2Configs:               oauthConfigs,
		RedirectToLogin:             false,
		DisableSessionExpiryRefresh: options.DeploymentValues.DisableSessionExpiryRefresh.Value(),
		Optional:                    false,
		SessionTokenFunc:            nil, // Default behavior
	})
	// Same as above but it redirects to the login page.
	apiKeyMiddlewareRedirect := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
		DB:                          options.Database,
		OAuth2Configs:               oauthConfigs,
		RedirectToLogin:             true,
		DisableSessionExpiryRefresh: options.DeploymentValues.DisableSessionExpiryRefresh.Value(),
		Optional:                    false,
		SessionTokenFunc:            nil, // Default behavior
	})
	// Same as the first but it's optional.
	apiKeyMiddlewareOptional := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
		DB:                          options.Database,
		OAuth2Configs:               oauthConfigs,
		RedirectToLogin:             false,
		DisableSessionExpiryRefresh: options.DeploymentValues.DisableSessionExpiryRefresh.Value(),
		Optional:                    true,
		SessionTokenFunc:            nil, // Default behavior
	})

	// API rate limit middleware. The counter is local and not shared between
	// replicas or instances of this middleware.
	apiRateLimiter := httpmw.RateLimit(options.APIRateLimit, time.Minute)

	derpHandler := derphttp.Handler(api.DERPServer)
	derpHandler, api.derpCloseFunc = tailnet.WithWebsocketSupport(api.DERPServer, derpHandler)
	cors := httpmw.Cors(options.DeploymentValues.Dangerous.AllowAllCors.Value())
	prometheusMW := httpmw.Prometheus(options.PrometheusRegistry)

	api.statsBatcher = options.StatsBatcher

	r.Use(
		httpmw.Recover(api.Logger),
		tracing.StatusWriterMiddleware,
		tracing.Middleware(api.TracerProvider),
		httpmw.AttachRequestID,
		httpmw.ExtractRealIP(api.RealIPConfig),
		httpmw.Logger(api.Logger),
		prometheusMW,
		// SubdomainAppMW checks if the first subdomain is a valid app URL. If
		// it is, it will serve that application.
		//
		// Workspace apps do their own auth and CORS and must be BEFORE the auth
		// and CORS middleware.
		api.workspaceAppServer.HandleSubdomain(apiRateLimiter),
		cors,
		// Build-Version is helpful for debugging.
		func(next http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Add("X-Coder-Build-Version", buildinfo.Version())
				next.ServeHTTP(w, r)
			})
		},
		// This header stops a browser from trying to MIME-sniff the content type and
		// forces it to stick with the declared content-type. This is the only valid
		// value for this header.
		// See: https://github.com/coder/security/issues/12
		func(next http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Add("X-Content-Type-Options", "nosniff")
				next.ServeHTTP(w, r)
			})
		},
		httpmw.CSRF(options.SecureAuthCookie),
	)

	r.Get("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("OK")) })

	// Attach workspace apps routes.
	r.Group(func(r chi.Router) {
		r.Use(apiRateLimiter)
		api.workspaceAppServer.Attach(r)
	})

	r.Route("/derp", func(r chi.Router) {
		r.Get("/", derpHandler.ServeHTTP)
		// This is used when UDP is blocked, and latency must be checked via HTTP(s).
		r.Get("/latency-check", func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
		})
	})

	// Register callback handlers for each OAuth2 provider.
	// We must support gitauth and externalauth for backwards compatibility.
	for _, route := range []string{"gitauth", "external-auth"} {
		r.Route("/"+route, func(r chi.Router) {
			for _, externalAuthConfig := range options.ExternalAuthConfigs {
				// We don't need to register a callback handler for device auth.
				if externalAuthConfig.DeviceAuth != nil {
					continue
				}
				r.Route(fmt.Sprintf("/%s/callback", externalAuthConfig.ID), func(r chi.Router) {
					r.Use(
						apiKeyMiddlewareRedirect,
						httpmw.ExtractOAuth2(externalAuthConfig, options.HTTPClient, nil),
					)
					r.Get("/", api.externalAuthCallback(externalAuthConfig))
				})
			}
		})
	}

	r.Route("/api/v2", func(r chi.Router) {
		api.APIHandler = r

		r.NotFound(func(rw http.ResponseWriter, r *http.Request) { httpapi.RouteNotFound(rw) })
		r.Use(
			// Specific routes can specify different limits, but every rate
			// limit must be configurable by the admin.
			apiRateLimiter,
			httpmw.ReportCLITelemetry(api.Logger, options.Telemetry),
		)
		r.Get("/", apiRoot)
		// All CSP errors will be logged
		r.Post("/csp/reports", api.logReportCSPViolations)

		r.Get("/buildinfo", buildInfo(api.AccessURL))
		// /regions is overridden in the enterprise version
		r.Group(func(r chi.Router) {
			r.Use(apiKeyMiddleware)
			r.Get("/regions", api.regions)
		})
		r.Route("/derp-map", func(r chi.Router) {
			// r.Use(apiKeyMiddleware)
			r.Get("/", api.derpMapUpdates)
		})
		r.Route("/deployment", func(r chi.Router) {
			r.Use(apiKeyMiddleware)
			r.Get("/config", api.deploymentValues)
			r.Get("/stats", api.deploymentStats)
			r.Get("/ssh", api.sshConfig)
		})
		r.Route("/experiments", func(r chi.Router) {
			r.Use(apiKeyMiddleware)
			r.Get("/available", handleExperimentsSafe)
			r.Get("/", api.handleExperimentsGet)
		})
		r.Get("/updatecheck", api.updateCheck)
		r.Route("/audit", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
			)

			r.Get("/", api.auditLogs)
			r.Post("/testgenerate", api.generateFakeAuditLog)
		})
		r.Route("/files", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				httpmw.RateLimit(options.FilesRateLimit, time.Minute),
			)
			r.Get("/{fileID}", api.fileByID)
			r.Post("/", api.postFile)
		})
		r.Route("/external-auth/{externalauth}", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				httpmw.ExtractExternalAuthParam(options.ExternalAuthConfigs),
			)
			r.Get("/", api.externalAuthByID)
			r.Post("/device", api.postExternalAuthDeviceByID)
			r.Get("/device", api.externalAuthDeviceByID)
		})
		r.Route("/organizations", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
			)
			r.Post("/", api.postOrganizations)
			r.Route("/{organization}", func(r chi.Router) {
				r.Use(
					httpmw.ExtractOrganizationParam(options.Database),
				)
				r.Get("/", api.organization)
				r.Post("/templateversions", api.postTemplateVersionsByOrganization)
				r.Route("/templates", func(r chi.Router) {
					r.Post("/", api.postTemplateByOrganization)
					r.Get("/", api.templatesByOrganization)
					r.Get("/examples", api.templateExamples)
					r.Route("/{templatename}", func(r chi.Router) {
						r.Get("/", api.templateByOrganizationAndName)
						r.Route("/versions/{templateversionname}", func(r chi.Router) {
							r.Get("/", api.templateVersionByOrganizationTemplateAndName)
							r.Get("/previous", api.previousTemplateVersionByOrganizationTemplateAndName)
						})
					})
				})
				r.Route("/members", func(r chi.Router) {
					r.Get("/roles", api.assignableOrgRoles)
					r.Route("/{user}", func(r chi.Router) {
						r.Use(
							httpmw.ExtractOrganizationMemberParam(options.Database),
						)
						r.Put("/roles", api.putMemberRoles)
						r.Post("/workspaces", api.postWorkspacesByOrganization)
					})
				})
			})
		})
		r.Route("/templates/{template}", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				httpmw.ExtractTemplateParam(options.Database),
			)
			r.Get("/daus", api.templateDAUs)
			r.Get("/", api.template)
			r.Delete("/", api.deleteTemplate)
			r.Patch("/", api.patchTemplateMeta)
			r.Route("/versions", func(r chi.Router) {
				r.Post("/archive", api.postArchiveTemplateVersions)
				r.Get("/", api.templateVersionsByTemplate)
				r.Patch("/", api.patchActiveTemplateVersion)
				r.Get("/{templateversionname}", api.templateVersionByName)
			})
		})
		r.Route("/templateversions/{templateversion}", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				httpmw.ExtractTemplateVersionParam(options.Database),
			)
			r.Get("/", api.templateVersion)
			r.Patch("/", api.patchTemplateVersion)
			r.Patch("/cancel", api.patchCancelTemplateVersion)
			r.Post("/archive", api.postArchiveTemplateVersion())
			r.Post("/unarchive", api.postUnarchiveTemplateVersion())
			// Old agents may expect a non-error response from /schema and /parameters endpoints.
			// The idea is to return an empty [], so that the coder CLI won't get blocked accidentally.
			r.Get("/schema", templateVersionSchemaDeprecated)
			r.Get("/parameters", templateVersionParametersDeprecated)
			r.Get("/rich-parameters", api.templateVersionRichParameters)
			r.Get("/external-auth", api.templateVersionExternalAuth)
			r.Get("/variables", api.templateVersionVariables)
			r.Get("/resources", api.templateVersionResources)
			r.Get("/logs", api.templateVersionLogs)
			r.Route("/dry-run", func(r chi.Router) {
				r.Post("/", api.postTemplateVersionDryRun)
				r.Get("/{jobID}", api.templateVersionDryRun)
				r.Get("/{jobID}/resources", api.templateVersionDryRunResources)
				r.Get("/{jobID}/logs", api.templateVersionDryRunLogs)
				r.Patch("/{jobID}/cancel", api.patchTemplateVersionDryRunCancel)
			})
		})
		r.Route("/users", func(r chi.Router) {
			r.Get("/first", api.firstUser)
			r.Post("/first", api.postFirstUser)
			r.Get("/authmethods", api.userAuthMethods)

			r.Group(func(r chi.Router) {
				// We use a tight limit for password login to protect against
				// audit-log write DoS, pbkdf2 DoS, and simple brute-force
				// attacks.
				//
				// This value is intentionally increased during tests.
				r.Use(httpmw.RateLimit(options.LoginRateLimit, time.Minute))
				r.Post("/login", api.postLogin)
				r.Route("/oauth2", func(r chi.Router) {
					r.Route("/github", func(r chi.Router) {
						r.Use(
							httpmw.ExtractOAuth2(options.GithubOAuth2Config, options.HTTPClient, nil),
						)
						r.Get("/callback", api.userOAuth2Github)
					})
				})
				r.Route("/oidc/callback", func(r chi.Router) {
					r.Use(
						httpmw.ExtractOAuth2(options.OIDCConfig, options.HTTPClient, oidcAuthURLParams),
					)
					r.Get("/", api.userOIDC)
				})
			})
			r.Group(func(r chi.Router) {
				r.Use(
					apiKeyMiddleware,
				)
				r.Post("/", api.postUser)
				r.Get("/", api.users)
				r.Post("/logout", api.postLogout)
				// These routes query information about site wide roles.
				r.Route("/roles", func(r chi.Router) {
					r.Get("/", api.assignableSiteRoles)
				})
				r.Route("/{user}", func(r chi.Router) {
					r.Use(httpmw.ExtractUserParam(options.Database))
					r.Post("/convert-login", api.postConvertLoginType)
					r.Delete("/", api.deleteUser)
					r.Get("/", api.userByName)
					r.Get("/login-type", api.userLoginType)
					r.Put("/profile", api.putUserProfile)
					r.Route("/status", func(r chi.Router) {
						r.Put("/suspend", api.putSuspendUserAccount())
						r.Put("/activate", api.putActivateUserAccount())
					})
					r.Route("/password", func(r chi.Router) {
						r.Put("/", api.putUserPassword)
					})
					// These roles apply to the site wide permissions.
					r.Put("/roles", api.putUserRoles)
					r.Get("/roles", api.userRoles)

					r.Route("/keys", func(r chi.Router) {
						r.Post("/", api.postAPIKey)
						r.Route("/tokens", func(r chi.Router) {
							r.Post("/", api.postToken)
							r.Get("/", api.tokens)
							r.Get("/tokenconfig", api.tokenConfig)
							r.Route("/{keyname}", func(r chi.Router) {
								r.Get("/", api.apiKeyByName)
							})
						})
						r.Route("/{keyid}", func(r chi.Router) {
							r.Get("/", api.apiKeyByID)
							r.Delete("/", api.deleteAPIKey)
						})
					})

					r.Route("/organizations", func(r chi.Router) {
						r.Get("/", api.organizationsByUser)
						r.Get("/{organizationname}", api.organizationByUserAndName)
					})
					r.Route("/workspace/{workspacename}", func(r chi.Router) {
						r.Get("/", api.workspaceByOwnerAndName)
						r.Get("/builds/{buildnumber}", api.workspaceBuildByBuildNumber)
					})
					r.Get("/gitsshkey", api.gitSSHKey)
					r.Put("/gitsshkey", api.regenerateGitSSHKey)
				})
			})
		})
		r.Route("/workspaceagents", func(r chi.Router) {
			r.Post("/azure-instance-identity", api.postWorkspaceAuthAzureInstanceIdentity)
			r.Post("/aws-instance-identity", api.postWorkspaceAuthAWSInstanceIdentity)
			r.Post("/google-instance-identity", api.postWorkspaceAuthGoogleInstanceIdentity)
			r.With(
				apiKeyMiddlewareOptional,
				httpmw.ExtractWorkspaceProxy(httpmw.ExtractWorkspaceProxyConfig{
					DB:       options.Database,
					Optional: true,
				}),
				httpmw.RequireAPIKeyOrWorkspaceProxyAuth(),
			).Get("/connection", api.workspaceAgentConnectionGeneric)
			r.Route("/me", func(r chi.Router) {
				r.Use(httpmw.ExtractWorkspaceAgent(httpmw.ExtractWorkspaceAgentConfig{
					DB:       options.Database,
					Optional: false,
				}))
				r.Get("/manifest", api.workspaceAgentManifest)
				// This route is deprecated and will be removed in a future release.
				// New agents will use /me/manifest instead.
				r.Get("/metadata", api.workspaceAgentManifest)
				r.Post("/startup", api.postWorkspaceAgentStartup)
				r.Patch("/startup-logs", api.patchWorkspaceAgentLogsDeprecated)
				r.Patch("/logs", api.patchWorkspaceAgentLogs)
				r.Post("/app-health", api.postWorkspaceAppHealth)
				// Deprecated: Required to support legacy agents
				r.Get("/gitauth", api.workspaceAgentsGitAuth)
				r.Get("/external-auth", api.workspaceAgentsExternalAuth)
				r.Get("/gitsshkey", api.agentGitSSHKey)
				r.Get("/coordinate", api.workspaceAgentCoordinate)
				r.Post("/report-stats", api.workspaceAgentReportStats)
				r.Post("/report-lifecycle", api.workspaceAgentReportLifecycle)
				r.Post("/metadata", api.workspaceAgentPostMetadata)
				r.Post("/metadata/{key}", api.workspaceAgentPostMetadataDeprecated)
			})
			r.Route("/{workspaceagent}", func(r chi.Router) {
				r.Use(
					// Allow either API key or external workspace proxy auth and require it.
					apiKeyMiddlewareOptional,
					httpmw.ExtractWorkspaceProxy(httpmw.ExtractWorkspaceProxyConfig{
						DB:       options.Database,
						Optional: true,
					}),
					httpmw.RequireAPIKeyOrWorkspaceProxyAuth(),

					httpmw.ExtractWorkspaceAgentParam(options.Database),
					httpmw.ExtractWorkspaceParam(options.Database),
				)
				r.Get("/", api.workspaceAgent)
				r.Get("/watch-metadata", api.watchWorkspaceAgentMetadata)
				r.Get("/startup-logs", api.workspaceAgentLogsDeprecated)
				r.Get("/logs", api.workspaceAgentLogs)
				r.Get("/listening-ports", api.workspaceAgentListeningPorts)
				r.Get("/connection", api.workspaceAgentConnection)
				r.Get("/coordinate", api.workspaceAgentClientCoordinate)

				// PTY is part of workspaceAppServer.
			})
		})
		r.Route("/workspaces", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
			)
			r.Get("/", api.workspaces)
			r.Route("/{workspace}", func(r chi.Router) {
				r.Use(
					httpmw.ExtractWorkspaceParam(options.Database),
				)
				r.Get("/", api.workspace)
				r.Patch("/", api.patchWorkspace)
				r.Route("/builds", func(r chi.Router) {
					r.Get("/", api.workspaceBuilds)
					r.Post("/", api.postWorkspaceBuilds)
				})
				r.Route("/autostart", func(r chi.Router) {
					r.Put("/", api.putWorkspaceAutostart)
				})
				r.Route("/ttl", func(r chi.Router) {
					r.Put("/", api.putWorkspaceTTL)
				})
				r.Get("/watch", api.watchWorkspace)
				r.Put("/extend", api.putExtendWorkspace)
				r.Put("/dormant", api.putWorkspaceDormant)
				r.Put("/autoupdates", api.putWorkspaceAutoupdates)
				r.Get("/resolve-autostart", api.resolveAutostart)
			})
		})
		r.Route("/workspacebuilds/{workspacebuild}", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				httpmw.ExtractWorkspaceBuildParam(options.Database),
				httpmw.ExtractWorkspaceParam(options.Database),
			)
			r.Get("/", api.workspaceBuild)
			r.Patch("/cancel", api.patchCancelWorkspaceBuild)
			r.Get("/logs", api.workspaceBuildLogs)
			r.Get("/parameters", api.workspaceBuildParameters)
			r.Get("/resources", api.workspaceBuildResources)
			r.Get("/state", api.workspaceBuildState)
		})
		r.Route("/authcheck", func(r chi.Router) {
			r.Use(apiKeyMiddleware)
			r.Post("/", api.checkAuthorization)
		})
		r.Route("/applications", func(r chi.Router) {
			r.Route("/host", func(r chi.Router) {
				// Don't leak the hostname to unauthenticated users.
				r.Use(apiKeyMiddleware)
				r.Get("/", api.appHost)
			})
			r.Route("/auth-redirect", func(r chi.Router) {
				// We want to redirect to login if they are not authenticated.
				r.Use(apiKeyMiddlewareRedirect)

				// This is a GET request as it's redirected to by the subdomain app
				// handler and the login page.
				r.Get("/", api.workspaceApplicationAuth)
			})
		})
		r.Route("/insights", func(r chi.Router) {
			r.Use(apiKeyMiddleware)
			r.Get("/daus", api.deploymentDAUs)
			r.Get("/user-activity", api.insightsUserActivity)
			r.Get("/user-latency", api.insightsUserLatency)
			r.Get("/templates", api.insightsTemplates)
		})
		r.Route("/debug", func(r chi.Router) {
			r.Use(
				apiKeyMiddleware,
				// Ensure only owners can access debug endpoints.
				func(next http.Handler) http.Handler {
					return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
						if !api.Authorize(r, rbac.ActionRead, rbac.ResourceDebugInfo) {
							httpapi.ResourceNotFound(rw)
							return
						}

						next.ServeHTTP(rw, r)
					})
				},
			)

			r.Get("/coordinator", api.debugCoordinator)
			r.Get("/health", api.debugDeploymentHealth)
			r.Get("/ws", (&healthcheck.WebsocketEchoServer{}).ServeHTTP)
		})
	})

	if options.SwaggerEndpoint {
		// Swagger UI requires the URL trailing slash. Otherwise, the browser tries to load /assets
		// from http://localhost:8080/assets instead of http://localhost:8080/swagger/assets.
		r.Get("/swagger", http.RedirectHandler("/swagger/", http.StatusTemporaryRedirect).ServeHTTP)
		// See globalHTTPSwaggerHandler comment as to why we use a package
		// global variable here.
		r.Get("/swagger/*", globalHTTPSwaggerHandler)
	}

	// Add CSP headers to all static assets and pages. CSP headers only affect
	// browsers, so these don't make sense on api routes.
	cspMW := httpmw.CSPHeaders(func() []string {
		if api.DeploymentValues.Dangerous.AllowAllCors {
			// In this mode, allow all external requests
			return []string{"*"}
		}
		if f := api.WorkspaceProxyHostsFn.Load(); f != nil {
			return (*f)()
		}
		// By default we do not add extra websocket connections to the CSP
		return []string{}
	})

	// Static file handler must be wrapped with HSTS handler if the
	// StrictTransportSecurityAge is set. We only need to set this header on
	// static files since it only affects browsers.
	r.NotFound(cspMW(compressHandler(httpmw.HSTS(api.SiteHandler, options.StrictTransportSecurityCfg))).ServeHTTP)

	// This must be before all middleware to improve the response time.
	// So make a new router, and mount the old one as the root.
	rootRouter := chi.NewRouter()
	// This is the only route we add before all the middleware.
	// We want to time the latency of the request, so any middleware will
	// interfere with that timing.
	rootRouter.Get("/latency-check", tracing.StatusWriterMiddleware(prometheusMW(LatencyCheck())).ServeHTTP)
	rootRouter.Mount("/", r)
	api.RootHandler = rootRouter

	return api
}

type API struct {
	// ctx is canceled immediately on shutdown, it can be used to abort
	// interruptible tasks.
	ctx    context.Context
	cancel context.CancelFunc

	// DeploymentID is loaded from the database on startup.
	DeploymentID string

	*Options
	// ID is a uniquely generated ID on initialization.
	// This is used to associate objects with a specific
	// Coder API instance, like workspace agents to a
	// specific replica.
	ID                                uuid.UUID
	Auditor                           atomic.Pointer[audit.Auditor]
	WorkspaceClientCoordinateOverride atomic.Pointer[func(rw http.ResponseWriter) bool]
	TailnetCoordinator                atomic.Pointer[tailnet.Coordinator]
	QuotaCommitter                    atomic.Pointer[proto.QuotaCommitter]
	// WorkspaceProxyHostsFn returns the hosts of healthy workspace proxies
	// for header reasons.
	WorkspaceProxyHostsFn atomic.Pointer[func() []string]
	// TemplateScheduleStore is a pointer to an atomic pointer because this is
	// passed to another struct, and we want them all to be the same reference.
	TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore]
	// UserQuietHoursScheduleStore is a pointer to an atomic pointer for the
	// same reason as TemplateScheduleStore.
	UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
	// DERPMapper mutates the DERPMap to include workspace proxies.
	DERPMapper atomic.Pointer[func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap]
	// AccessControlStore is a pointer to an atomic pointer since it is
	// passed to dbauthz.
	AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore]

	HTTPAuth *HTTPAuthorizer

	// APIHandler serves "/api/v2"
	APIHandler chi.Router
	// RootHandler serves "/"
	RootHandler chi.Router

	// SiteHandler serves static files for the dashboard.
	SiteHandler *site.Handler

	WebsocketWaitMutex sync.Mutex
	WebsocketWaitGroup sync.WaitGroup
	derpCloseFunc      func()

	metricsCache          *metricscache.Cache
	updateChecker         *updatecheck.Checker
	WorkspaceAppsProvider workspaceapps.SignedTokenProvider
	workspaceAppServer    *workspaceapps.Server
	agentProvider         workspaceapps.AgentProvider

	// Experiments contains the list of experiments currently enabled.
	// This is used to gate features that are not yet ready for production.
	Experiments codersdk.Experiments

	healthCheckGroup *singleflight.Group[string, *healthcheck.Report]
	healthCheckCache atomic.Pointer[healthcheck.Report]

	statsBatcher *batchstats.Batcher

	Acquirer *provisionerdserver.Acquirer
}

// Close waits for all WebSocket connections to drain before returning.
func (api *API) Close() error {
	api.cancel()
	api.derpCloseFunc()

	api.WebsocketWaitMutex.Lock()
	api.WebsocketWaitGroup.Wait()
	api.WebsocketWaitMutex.Unlock()

	api.metricsCache.Close()
	if api.updateChecker != nil {
		api.updateChecker.Close()
	}
	_ = api.workspaceAppServer.Close()
	coordinator := api.TailnetCoordinator.Load()
	if coordinator != nil {
		_ = (*coordinator).Close()
	}
	_ = api.agentProvider.Close()
	return nil
}

func compressHandler(h http.Handler) http.Handler {
	level := 5
	if flag.Lookup("test.v") != nil {
		level = 1
	}

	cmp := middleware.NewCompressor(level,
		"text/*",
		"application/*",
		"image/*",
	)
	cmp.SetEncoder("br", func(w io.Writer, level int) io.Writer {
		return brotli.NewWriterLevel(w, level)
	})
	cmp.SetEncoder("zstd", func(w io.Writer, level int) io.Writer {
		zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level)))
		if err != nil {
			panic("invalid zstd compressor: " + err.Error())
		}
		return zw
	})

	return cmp.Handler(h)
}

// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd.
// Useful when starting coderd and provisionerd in the same process.
func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) {
	tracer := api.TracerProvider.Tracer(tracing.TracerName)
	clientSession, serverSession := provisionersdk.MemTransportPipe()
	defer func() {
		if err != nil {
			_ = clientSession.Close()
			_ = serverSession.Close()
		}
	}()

	tags := provisionerdserver.Tags{
		provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
	}

	mux := drpcmux.New()
	name := namesgenerator.GetRandomName(1)
	logger := api.Logger.Named(fmt.Sprintf("inmem-provisionerd-%s", name))
	logger.Info(ctx, "starting in-memory provisioner daemon")
	srv, err := provisionerdserver.NewServer(
		api.ctx,
		api.AccessURL,
		uuid.New(),
		logger,
		[]database.ProvisionerType{
			database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform,
		},
		tags,
		api.Database,
		api.Pubsub,
		api.Acquirer,
		api.Telemetry,
		tracer,
		&api.QuotaCommitter,
		&api.Auditor,
		api.TemplateScheduleStore,
		api.UserQuietHoursScheduleStore,
		api.DeploymentValues,
		provisionerdserver.Options{
			OIDCConfig:          api.OIDCConfig,
			ExternalAuthConfigs: api.ExternalAuthConfigs,
		},
	)
	if err != nil {
		return nil, err
	}
	err = proto.DRPCRegisterProvisionerDaemon(mux, srv)
	if err != nil {
		return nil, err
	}
	server := drpcserver.NewWithOptions(&tracing.DRPCHandler{Handler: mux},
		drpcserver.Options{
			Log: func(err error) {
				if xerrors.Is(err, io.EOF) {
					return
				}
				logger.Debug(ctx, "drpc server error", slog.Error(err))
			},
		},
	)
	go func() {
		err := server.Serve(ctx, serverSession)
		logger.Info(ctx, "provisioner daemon disconnected", slog.Error(err))
		// close the sessions, so we don't leak goroutines serving them.
		_ = clientSession.Close()
		_ = serverSession.Close()
	}()

	return proto.NewDRPCProvisionerDaemonClient(clientSession), nil
}

func (api *API) DERPMap() *tailcfg.DERPMap {
	fn := api.DERPMapper.Load()
	if fn != nil {
		return (*fn)(api.Options.BaseDERPMap)
	}

	return api.Options.BaseDERPMap
}

// nolint:revive
func ReadExperiments(log slog.Logger, raw []string) codersdk.Experiments {
	exps := make([]codersdk.Experiment, 0, len(raw))
	for _, v := range raw {
		switch v {
		case "*":
			exps = append(exps, codersdk.ExperimentsAll...)
		default:
			ex := codersdk.Experiment(strings.ToLower(v))
			if !slice.Contains(codersdk.ExperimentsAll, ex) {
				log.Warn(context.Background(), "🐉 HERE BE DRAGONS: opting into hidden experiment", slog.F("experiment", ex))
			}
			exps = append(exps, ex)
		}
	}
	return exps
}